{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read Data Sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-23T20:25:37.655915Z",
     "start_time": "2017-06-23T20:25:37.211540Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import time\n",
    "from collections import namedtuple\n",
    "pd.set_option(\"display.max_rows\",35)\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-23T20:25:37.922753Z",
     "start_time": "2017-06-23T20:25:37.657487Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class dataset:\n",
    "    kdd_train_2labels = pd.read_pickle(\"dataset/kdd_train_2labels.pkl\")\n",
    "    kdd_test_2labels = pd.read_pickle(\"dataset/kdd_test_2labels.pkl\")\n",
    "    kdd_test__2labels = pd.read_pickle(\"dataset/kdd_test__2labels.pkl\")\n",
    "    \n",
    "    kdd_train_5labels = pd.read_pickle(\"dataset/kdd_train_5labels.pkl\")\n",
    "    kdd_test_5labels = pd.read_pickle(\"dataset/kdd_test_5labels.pkl\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-23T20:25:37.929594Z",
     "start_time": "2017-06-23T20:25:37.924531Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(125973, 124)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.kdd_train_2labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-23T20:25:37.937528Z",
     "start_time": "2017-06-23T20:25:37.931031Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(22544, 124)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.kdd_test_2labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-23T20:25:38.794163Z",
     "start_time": "2017-06-23T20:25:37.938962Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.99589320646770185"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import model_selection as ms\n",
    "from sklearn import preprocessing as pp\n",
    "\n",
    "class preprocess:\n",
    "    \n",
    "    output_columns_2labels = ['is_Normal','is_Attack']\n",
    "    \n",
    "    x_input = dataset.kdd_train_2labels.drop(output_columns_2labels, axis = 1)\n",
    "    y_output = dataset.kdd_train_2labels.loc[:,output_columns_2labels]\n",
    "\n",
    "    x_test_input = dataset.kdd_test_2labels.drop(output_columns_2labels, axis = 1)\n",
    "    y_test = dataset.kdd_test_2labels.loc[:,output_columns_2labels]\n",
    "    \n",
    "    x_test__input = dataset.kdd_test__2labels.drop(output_columns_2labels, axis = 1)\n",
    "    y_test_ = dataset.kdd_test__2labels.loc[:,output_columns_2labels]\n",
    "\n",
    "    ss = pp.StandardScaler()\n",
    "\n",
    "    x_train = ss.fit_transform(x_input)\n",
    "    x_test = ss.transform(x_test_input)\n",
    "    x_test_ = ss.transform(x_test__input)\n",
    "\n",
    "    y_train = y_output.values\n",
    "    y_test = y_test.values\n",
    "    y_test_ = y_test_.values\n",
    "\n",
    "    \n",
    "preprocess.x_train.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-23T20:25:40.647390Z",
     "start_time": "2017-06-23T20:25:38.795614Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.contrib.legacy_seq2seq.python.ops.seq2seq import basic_rnn_seq2seq\n",
    "from tensorflow.contrib.rnn import RNNCell, GRUCell, MultiRNNCell\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-23T20:25:40.902198Z",
     "start_time": "2017-06-23T20:25:40.648899Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class network(object):\n",
    "    \n",
    "    input_dim = 122\n",
    "    classes = 2\n",
    "    hidden_encoder_dim = 122\n",
    "    hidden_layers = 1\n",
    "    latent_dim = 10\n",
    "\n",
    "    hidden_decoder_dim = 122\n",
    "    lam = 0.01\n",
    "    \n",
    "    def __init__(self, classes, hidden_layers, num_of_features):\n",
    "        self.classes = classes\n",
    "        self.hidden_layers = hidden_layers\n",
    "        self.latent_dim = num_of_features\n",
    "            \n",
    "    def build_layers(self):\n",
    "        tf.reset_default_graph()\n",
    "        #learning_rate = tf.Variable(initial_value=0.001)\n",
    "\n",
    "        input_dim = self.input_dim\n",
    "        classes = self.classes\n",
    "        hidden_encoder_dim = self.hidden_encoder_dim\n",
    "        hidden_layers = self.hidden_layers\n",
    "        latent_dim = self.latent_dim\n",
    "        hidden_decoder_dim = self.hidden_decoder_dim\n",
    "        lam = self.lam\n",
    "        \n",
    "        with tf.variable_scope(\"Input\"):\n",
    "            self.x_input = tf.placeholder(\"float\", shape=[None, 1, input_dim])\n",
    "            self.y_input_ = tf.placeholder(\"float\", shape=[None, 1, classes])\n",
    "            self.keep_prob = tf.placeholder(\"float\")\n",
    "            self.lr = tf.placeholder(\"float\")\n",
    "            self.x_list = tf.unstack(self.x_input, axis= 1)\n",
    "            self.y_list_ = tf.unstack(self.y_input_, axis = 1)\n",
    "            self.y_ = self.y_list_[0]\n",
    "            \n",
    "            #GO = tf.fill((tf.shape(self.x)[0], 1), 0.5)\n",
    "            \n",
    "            #y_with_GO = tf.stack([self.y_, GO])\n",
    "            \n",
    "        with tf.variable_scope(\"lstm\"):\n",
    "            multi_cell = MultiRNNCell([GRUCell(input_dim) for i in range(hidden_layers)] )\n",
    "            \n",
    "            self.y, states = basic_rnn_seq2seq(self.x_list, self.y_list_, multi_cell)\n",
    "            #self.y = tf.slice(self.y, [0, 0], [-1,2])\n",
    "            \n",
    "            #self.out = tf.squeeze(self.y)\n",
    "            \n",
    "            #self.y = tf.layers.dense(self.y[0], classes, activation = None)\n",
    "            \n",
    "            self.y = tf.slice(self.y[0], [0, 0], [-1,2])\n",
    "            \n",
    "        with tf.variable_scope(\"Loss\"):\n",
    "            \n",
    "            self.regularized_loss = tf.losses.mean_squared_error(self.y_, self.y)\n",
    "            correct_prediction = tf.equal(tf.argmax(self.y_, 1), tf.argmax(self.y, 1))\n",
    "            self.tf_accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name = \"Accuracy\")\n",
    "\n",
    "        with tf.variable_scope(\"Optimizer\"):\n",
    "            learning_rate=self.lr\n",
    "            optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "            gradients, variables = zip(*optimizer.compute_gradients(self.regularized_loss))\n",
    "            gradients = [\n",
    "                None if gradient is None else tf.clip_by_value(gradient, -1, 1)\n",
    "                for gradient in gradients]\n",
    "            self.train_op = optimizer.apply_gradients(zip(gradients, variables))\n",
    "            #self.train_op = optimizer.minimize(self.regularized_loss)\n",
    "            \n",
    "        # add op for merging summary\n",
    "        #self.summary_op = tf.summary.merge_all()\n",
    "        self.pred = tf.argmax(self.y, axis = 1)\n",
    "        self.actual = tf.argmax(self.y_, axis = 1)\n",
    "\n",
    "        # add Saver ops\n",
    "        self.saver = tf.train.Saver()\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-01T00:59:00.684124Z",
     "start_time": "2017-06-01T00:58:59.843181Z"
    }
   },
   "source": [
    "batch_iterations = 200\n",
    "\n",
    "x_train, x_valid, y_train, y_valid, = ms.train_test_split(preprocess.x_train, \n",
    "                                                                          preprocess.y_train, \n",
    "                                                                          test_size=0.1)\n",
    "batch_indices = np.array_split(np.arange(x_train.shape[0]), \n",
    "                                           batch_iterations)\n",
    "                                                                          \n",
    "for i in batch_indices:\n",
    "    print(x_train[i,np.newaxis,:])\n",
    "    print(y_train[i,np.newaxis,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-23T20:25:41.198524Z",
     "start_time": "2017-06-23T20:25:40.903792Z"
    },
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import collections\n",
    "\n",
    "class Train:    \n",
    "    \n",
    "    result = namedtuple(\"score\", ['epoch', 'no_of_features','hidden_layers','train_score', 'test_score', 'test_score_20', 'time_taken'])\n",
    "\n",
    "    predictions = {}\n",
    "\n",
    "    results = []\n",
    "    best_acc = 0\n",
    "    best_acc_global = 0\n",
    "\n",
    "    def train(epochs, net, h,f, lrs):\n",
    "        batch_iterations = 200\n",
    "        train_loss = None\n",
    "        Train.best_acc = 0\n",
    "        os.makedirs(\"dataset/tf_GRU_nsl_kdd-orig/hidden layers_{}_features count_{}\".format(h,f),\n",
    "                    exist_ok = True)\n",
    "        with tf.Session() as sess:\n",
    "            #summary_writer_train = tf.summary.FileWriter('./logs/kdd/VAE/training', graph=sess.graph)\n",
    "            #summary_writer_valid = tf.summary.FileWriter('./logs/kdd/VAE/validation')\n",
    "\n",
    "            sess.run(tf.global_variables_initializer())\n",
    "            start_time = time.perf_counter()\n",
    "            \n",
    "            accuracy, pred_value, actual_value, y_pred = sess.run([net.tf_accuracy, \n",
    "                                                                   net.pred, \n",
    "                                                                   net.actual, net.y], \n",
    "                                                                  feed_dict={net.x_input: preprocess.x_test[:,np.newaxis,:], \n",
    "                                                                             net.y_input_: preprocess.y_test[:,np.newaxis,:], \n",
    "                                                                             net.keep_prob:1})\n",
    "            \n",
    "            print(\"Initial Accuracy, before training: {}\".format(accuracy))\n",
    "            \n",
    "            for c, lr in enumerate(lrs):\n",
    "                for epoch in range(1, (epochs+1)):\n",
    "                    x_train, x_valid, y_train, y_valid, = ms.train_test_split(preprocess.x_train, \n",
    "                                                                              preprocess.y_train, \n",
    "                                                                              test_size=0.1)\n",
    "                    batch_indices = np.array_split(np.arange(x_train.shape[0]), \n",
    "                                               batch_iterations)\n",
    "\n",
    "                    for i in batch_indices:\n",
    "\n",
    "                        _, train_loss = sess.run([net.train_op, net.regularized_loss], #net.summary_op\n",
    "                                                              feed_dict={net.x_input: x_train[i,np.newaxis,:], \n",
    "                                                                         net.y_input_: y_train[i,np.newaxis,:], \n",
    "                                                                         net.keep_prob:1, net.lr:lr})\n",
    "                        #summary_writer_train.add_summary(summary_str, epoch)\n",
    "                        if(train_loss > 1e9):\n",
    "                            print(\"Step {} | Training Loss: {:.6f}\".format(epoch, train_loss))\n",
    "\n",
    "\n",
    "                    valid_accuracy,valid_loss = sess.run([net.tf_accuracy, net.regularized_loss], #net.summary_op \n",
    "                                                          feed_dict={net.x_input: x_valid[:,np.newaxis,:], \n",
    "                                                                     net.y_input_: y_valid[:,np.newaxis,:], \n",
    "                                                                     net.keep_prob:1, net.lr:lr})\n",
    "                    #summary_writer_valid.add_summary(summary_str, epoch)\n",
    "\n",
    "\n",
    "\n",
    "                    accuracy, pred_value, actual_value, y_pred = sess.run([net.tf_accuracy, \n",
    "                                                                   net.pred, \n",
    "                                                                   net.actual, net.y], \n",
    "                                                                  feed_dict={net.x_input: preprocess.x_test[:,np.newaxis,:], \n",
    "                                                                             net.y_input_: preprocess.y_test[:,np.newaxis,:], \n",
    "                                                                             net.keep_prob:1, net.lr:lr})\n",
    "                    \n",
    "                    accuracy_, pred_value_, actual_value_, y_pred_ = sess.run([net.tf_accuracy, \n",
    "                                                                   net.pred, \n",
    "                                                                   net.actual, net.y], \n",
    "                                                                  feed_dict={net.x_input: preprocess.x_test_[:,np.newaxis,:], \n",
    "                                                                             net.y_input_: preprocess.y_test_[:,np.newaxis,:], \n",
    "                                                                             net.keep_prob:1, net.lr:lr})\n",
    "\n",
    "                    print(\"Step {} | Training Loss: {:.6f} | Train Accuracy: {:.6f} | Test Accuracy: {:.6f}, {:.6f}\".format(epoch, train_loss, valid_accuracy, accuracy, accuracy_))\n",
    "\n",
    "                    if accuracy > Train.best_acc_global:\n",
    "                                Train.best_acc_global = accuracy\n",
    "                                Train.pred_value = pred_value\n",
    "                                Train.actual_value = actual_value\n",
    "                                Train.pred_value_ = pred_value_\n",
    "                                Train.actual_value_ = actual_value_\n",
    "                                Train.best_parameters = \"Hidden Layers:{}, Features Count:{}\".format(h, f)\n",
    "\n",
    "                    if accuracy > Train.best_acc:\n",
    "\n",
    "                        #net.saver.save(sess, \"dataset/tf_vae_only_nsl_kdd_hidden layers_{}_features count_{}\".format(epochs,h,f))\n",
    "                        #Train.results.append(Train.result(epochs, f, h,valid_accuracy, accuracy))\n",
    "                        #curr_pred = pd.DataFrame({\"Attack_prob\":y_pred[:,-2], \"Normal_prob\":y_pred[:, -1]})\n",
    "                        #Train.predictions.update({\"{}_{}_{}\".format(epochs,f,h):curr_pred})\n",
    "\n",
    "                        Train.best_acc = accuracy\n",
    "                        if not (np.isnan(train_loss)):\n",
    "                            net.saver.save(sess, \n",
    "                                       \"dataset/tf_GRU_nsl_kdd-orig/hidden layers_{}_features count_{}/model\"\n",
    "                                       .format(h,f), \n",
    "                                       global_step = epoch, \n",
    "                                       write_meta_graph=False)\n",
    "\n",
    "                        curr_pred = pd.DataFrame({\"Attack_prob\":y_pred[:,-2], \"Normal_prob\":y_pred[:, -1], \"Prediction\":pred_value})\n",
    "                        Train.predictions.update({\"{}_{}_{}\".format((epochs+1)* (c+1),f,h):\n",
    "                                                  (curr_pred, \n",
    "                                                   Train.result((epochs+1)*(c+1), f, h,valid_accuracy, accuracy, accuracy_, time.perf_counter() - start_time))})\n",
    "\n",
    "\n",
    "\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-23T20:25:41.260108Z",
     "start_time": "2017-06-23T20:25:41.200370Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import itertools\n",
    "class Hyperparameters:\n",
    "#    features_arr = [2, 4, 8, 16, 32, 64, 128, 256]\n",
    "#    hidden_layers_arr = [2, 4, 6, 10]\n",
    "\n",
    "    def start_training():\n",
    "\n",
    "        global df_results\n",
    "        global past_scores\n",
    "        \n",
    "        Train.predictions = {}\n",
    "        Train.results = []\n",
    "        \n",
    "        features_arr = [1] #[4, 8, 16, 32]\n",
    "        hidden_layers_arr = [1, 3, 5]\n",
    "\n",
    "        epochs = [5]\n",
    "        lrs = [1e-2, 1e-2/2, 1e-2/4]\n",
    "\n",
    "        for e, h, f in itertools.product(epochs, hidden_layers_arr, features_arr):\n",
    "            print(\"Current Layer Attributes - epochs:{} hidden layers:{} features count:{}\".format(e,h,f))\n",
    "            n = network(2,h,f)\n",
    "            n.build_layers()\n",
    "            Train.train(e, n, h,f, lrs)\n",
    "            \n",
    "        dict1 = {}\n",
    "        dict2 = []\n",
    "        for k, (v1, v2) in Train.predictions.items():\n",
    "            dict1.update({k: v1})\n",
    "            dict2.append(v2)\n",
    "            \n",
    "        Train.predictions = dict1\n",
    "        Train.results = dict2\n",
    "        df_results = pd.DataFrame(Train.results)\n",
    "        temp = df_results.set_index(['no_of_features', 'hidden_layers'])\n",
    "\n",
    "        if not os.path.isfile('dataset/tf_GRU_nsl_kdd-orig_all.pkl'):\n",
    "            past_scores = temp\n",
    "        else:\n",
    "            past_scores = pd.read_pickle(\"dataset/tf_GRU_nsl_kdd-orig_all.pkl\")\n",
    "\n",
    "        past_scores.append(temp).to_pickle(\"dataset/tf_GRU_nsl_kdd-orig_all.pkl\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-23T21:43:45.563495Z",
     "start_time": "2017-06-23T20:25:41.261589Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Layer Attributes - epochs:5 hidden layers:1 features count:1\n",
      "Initial Accuracy, before training: 0.7715134620666504\n",
      "Step 1 | Training Loss: 0.002247 | Train Accuracy: 0.997857 | Test Accuracy: 0.800124, 0.619747\n",
      "Step 2 | Training Loss: 0.001798 | Train Accuracy: 0.999524 | Test Accuracy: 0.878815, 0.769451\n",
      "Step 3 | Training Loss: 0.000320 | Train Accuracy: 0.999921 | Test Accuracy: 0.913591, 0.835612\n",
      "Step 4 | Training Loss: 0.000357 | Train Accuracy: 1.000000 | Test Accuracy: 0.932798, 0.872152\n",
      "Step 5 | Training Loss: 0.000136 | Train Accuracy: 1.000000 | Test Accuracy: 0.937278, 0.880675\n",
      "Step 1 | Training Loss: 0.000078 | Train Accuracy: 1.000000 | Test Accuracy: 0.955465, 0.915274\n",
      "Step 2 | Training Loss: 0.000383 | Train Accuracy: 1.000000 | Test Accuracy: 0.957417, 0.918987\n",
      "Step 3 | Training Loss: 0.000069 | Train Accuracy: 1.000000 | Test Accuracy: 0.957505, 0.919156\n",
      "Step 4 | Training Loss: 0.000093 | Train Accuracy: 1.000000 | Test Accuracy: 0.957727, 0.919578\n",
      "Step 5 | Training Loss: 0.000049 | Train Accuracy: 1.000000 | Test Accuracy: 0.957638, 0.919409\n",
      "Step 1 | Training Loss: 0.000055 | Train Accuracy: 1.000000 | Test Accuracy: 0.957816, 0.919747\n",
      "Step 2 | Training Loss: 0.000026 | Train Accuracy: 1.000000 | Test Accuracy: 0.957993, 0.920084\n",
      "Step 3 | Training Loss: 0.000047 | Train Accuracy: 1.000000 | Test Accuracy: 0.958038, 0.920169\n",
      "Step 4 | Training Loss: 0.000016 | Train Accuracy: 1.000000 | Test Accuracy: 0.958171, 0.920422\n",
      "Step 5 | Training Loss: 0.000026 | Train Accuracy: 1.000000 | Test Accuracy: 0.958215, 0.920506\n",
      "Current Layer Attributes - epochs:5 hidden layers:3 features count:1\n",
      "Initial Accuracy, before training: 0.3616483211517334\n",
      "Step 1 | Training Loss: 0.000003 | Train Accuracy: 0.999841 | Test Accuracy: 0.998181, 0.996540\n",
      "Step 2 | Training Loss: 0.000182 | Train Accuracy: 0.999762 | Test Accuracy: 0.977954, 0.958059\n",
      "Step 3 | Training Loss: 0.000003 | Train Accuracy: 0.999921 | Test Accuracy: 0.999423, 0.998903\n",
      "Step 4 | Training Loss: 0.000001 | Train Accuracy: 1.000000 | Test Accuracy: 0.999823, 0.999662\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999867, 0.999747\n",
      "Step 1 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999911, 0.999831\n",
      "Step 2 | Training Loss: 0.000001 | Train Accuracy: 1.000000 | Test Accuracy: 0.999911, 0.999831\n",
      "Step 3 | Training Loss: 0.000004 | Train Accuracy: 1.000000 | Test Accuracy: 0.999911, 0.999831\n",
      "Step 4 | Training Loss: 0.000001 | Train Accuracy: 1.000000 | Test Accuracy: 0.999911, 0.999831\n",
      "Step 5 | Training Loss: 0.000004 | Train Accuracy: 1.000000 | Test Accuracy: 0.999556, 0.999156\n",
      "Step 1 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999823, 0.999662\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999823, 0.999662\n",
      "Step 3 | Training Loss: 0.000800 | Train Accuracy: 1.000000 | Test Accuracy: 0.999823, 0.999662\n",
      "Step 4 | Training Loss: 0.000001 | Train Accuracy: 1.000000 | Test Accuracy: 0.999823, 0.999662\n",
      "Step 5 | Training Loss: 0.000001 | Train Accuracy: 1.000000 | Test Accuracy: 0.999823, 0.999662\n",
      "Current Layer Attributes - epochs:5 hidden layers:5 features count:1\n",
      "Initial Accuracy, before training: 0.5675567984580994\n",
      "Step 1 | Training Loss: 0.020722 | Train Accuracy: 0.972218 | Test Accuracy: 0.760335, 0.546751\n",
      "Step 2 | Training Loss: 0.020305 | Train Accuracy: 0.978489 | Test Accuracy: 0.772489, 0.568354\n",
      "Step 3 | Training Loss: 0.013774 | Train Accuracy: 0.977457 | Test Accuracy: 0.750444, 0.525992\n",
      "Step 4 | Training Loss: 0.026087 | Train Accuracy: 0.978171 | Test Accuracy: 0.776304, 0.574430\n",
      "Step 5 | Training Loss: 0.005209 | Train Accuracy: 0.997460 | Test Accuracy: 0.894029, 0.798397\n",
      "Step 1 | Training Loss: 0.001756 | Train Accuracy: 0.997460 | Test Accuracy: 0.904143, 0.817637\n",
      "Step 2 | Training Loss: 0.003468 | Train Accuracy: 0.999047 | Test Accuracy: 0.899885, 0.809536\n",
      "Step 3 | Training Loss: 0.002058 | Train Accuracy: 0.999127 | Test Accuracy: 0.908268, 0.825485\n",
      "Step 4 | Training Loss: 0.000006 | Train Accuracy: 0.999841 | Test Accuracy: 0.959945, 0.923797\n",
      "Step 5 | Training Loss: 0.000029 | Train Accuracy: 0.999762 | Test Accuracy: 0.962473, 0.928608\n",
      "Step 1 | Training Loss: 0.001767 | Train Accuracy: 0.999682 | Test Accuracy: 0.962340, 0.928354\n",
      "Step 2 | Training Loss: 0.000057 | Train Accuracy: 0.999762 | Test Accuracy: 0.961852, 0.927426\n",
      "Step 3 | Training Loss: 0.000251 | Train Accuracy: 0.999841 | Test Accuracy: 0.961985, 0.927679\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 0.999762 | Test Accuracy: 0.962074, 0.927848\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.972498, 0.947679\n",
      "Current Layer Attributes - epochs:5 hidden layers:1 features count:1\n",
      "Initial Accuracy, before training: 0.6001153588294983\n",
      "Step 1 | Training Loss: 0.006357 | Train Accuracy: 0.990713 | Test Accuracy: 0.804161, 0.627426\n",
      "Step 2 | Training Loss: 0.000876 | Train Accuracy: 0.999762 | Test Accuracy: 0.900373, 0.810464\n",
      "Step 3 | Training Loss: 0.000928 | Train Accuracy: 0.999841 | Test Accuracy: 0.925080, 0.857468\n",
      "Step 4 | Training Loss: 0.000142 | Train Accuracy: 0.999921 | Test Accuracy: 0.934972, 0.876287\n",
      "Step 5 | Training Loss: 0.000247 | Train Accuracy: 0.999841 | Test Accuracy: 0.937278, 0.880675\n",
      "Step 1 | Training Loss: 0.000080 | Train Accuracy: 0.999841 | Test Accuracy: 0.938520, 0.883038\n",
      "Step 2 | Training Loss: 0.000068 | Train Accuracy: 1.000000 | Test Accuracy: 0.939141, 0.884219\n",
      "Step 3 | Training Loss: 0.000042 | Train Accuracy: 1.000000 | Test Accuracy: 0.939319, 0.884557\n",
      "Step 4 | Training Loss: 0.000091 | Train Accuracy: 0.999921 | Test Accuracy: 0.940206, 0.886245\n",
      "Step 5 | Training Loss: 0.000976 | Train Accuracy: 0.999841 | Test Accuracy: 0.940339, 0.886498\n",
      "Step 1 | Training Loss: 0.000063 | Train Accuracy: 0.999682 | Test Accuracy: 0.941004, 0.887764\n",
      "Step 2 | Training Loss: 0.000052 | Train Accuracy: 0.999841 | Test Accuracy: 0.958747, 0.921519\n",
      "Step 3 | Training Loss: 0.000053 | Train Accuracy: 1.000000 | Test Accuracy: 0.958747, 0.921519\n",
      "Step 4 | Training Loss: 0.000011 | Train Accuracy: 0.999921 | Test Accuracy: 0.958481, 0.921013\n",
      "Step 5 | Training Loss: 0.000906 | Train Accuracy: 0.999921 | Test Accuracy: 0.958614, 0.921266\n",
      "Current Layer Attributes - epochs:5 hidden layers:3 features count:1\n",
      "Initial Accuracy, before training: 0.6293026804924011\n",
      "Step 1 | Training Loss: 0.000011 | Train Accuracy: 0.999921 | Test Accuracy: 0.999690, 0.999409\n",
      "Step 2 | Training Loss: 0.000003 | Train Accuracy: 0.999921 | Test Accuracy: 0.999512, 0.999072\n",
      "Step 3 | Training Loss: 0.000001 | Train Accuracy: 0.999841 | Test Accuracy: 0.999512, 0.999072\n",
      "Step 4 | Training Loss: 0.000001 | Train Accuracy: 0.999921 | Test Accuracy: 0.999690, 0.999409\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 0.999444 | Test Accuracy: 0.998802, 0.998987\n",
      "Step 1 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999734, 0.999494\n",
      "Step 2 | Training Loss: 0.000001 | Train Accuracy: 0.999841 | Test Accuracy: 0.999202, 0.999409\n",
      "Step 3 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999778, 0.999578\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999734, 0.999494\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999157, 0.998397\n",
      "Step 1 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999157, 0.998397\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999246, 0.998565\n",
      "Step 3 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999335, 0.998734\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999379, 0.998819\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999246, 0.998565\n",
      "Current Layer Attributes - epochs:5 hidden layers:5 features count:1\n",
      "Initial Accuracy, before training: 0.6438519954681396\n",
      "Step 1 | Training Loss: 0.011850 | Train Accuracy: 0.985791 | Test Accuracy: 0.814008, 0.647257\n",
      "Step 2 | Training Loss: 0.007320 | Train Accuracy: 0.986506 | Test Accuracy: 0.823146, 0.665232\n",
      "Step 3 | Training Loss: 0.016808 | Train Accuracy: 0.984998 | Test Accuracy: 0.822303, 0.662954\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 4 | Training Loss: 0.005629 | Train Accuracy: 0.994205 | Test Accuracy: 0.910220, 0.829198\n",
      "Step 5 | Training Loss: 0.000003 | Train Accuracy: 0.999682 | Test Accuracy: 0.973962, 0.950464\n",
      "Step 1 | Training Loss: 0.000012 | Train Accuracy: 0.999682 | Test Accuracy: 0.984120, 0.969789\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 0.999841 | Test Accuracy: 0.974273, 0.951055\n",
      "Step 3 | Training Loss: 0.000001 | Train Accuracy: 0.999762 | Test Accuracy: 0.975515, 0.953502\n",
      "Step 4 | Training Loss: 0.000001 | Train Accuracy: 0.999762 | Test Accuracy: 0.974450, 0.951392\n",
      "Step 5 | Training Loss: 0.000011 | Train Accuracy: 0.999682 | Test Accuracy: 0.974583, 0.951646\n",
      "Step 1 | Training Loss: 0.001774 | Train Accuracy: 0.999603 | Test Accuracy: 0.984652, 0.970802\n",
      "Step 2 | Training Loss: 0.000001 | Train Accuracy: 1.000000 | Test Accuracy: 0.984741, 0.971055\n",
      "Step 3 | Training Loss: 0.000012 | Train Accuracy: 1.000000 | Test Accuracy: 0.985096, 0.971730\n",
      "Step 4 | Training Loss: 0.000061 | Train Accuracy: 1.000000 | Test Accuracy: 0.984608, 0.970802\n",
      "Step 5 | Training Loss: 0.000001 | Train Accuracy: 0.999682 | Test Accuracy: 0.984697, 0.970970\n",
      "Current Layer Attributes - epochs:5 hidden layers:1 features count:1\n",
      "Initial Accuracy, before training: 0.6022888422012329\n",
      "Step 1 | Training Loss: 0.004444 | Train Accuracy: 0.997381 | Test Accuracy: 0.797418, 0.614599\n",
      "Step 2 | Training Loss: 0.000639 | Train Accuracy: 0.999682 | Test Accuracy: 0.838627, 0.692996\n",
      "Step 3 | Training Loss: 0.000394 | Train Accuracy: 1.000000 | Test Accuracy: 0.895759, 0.801688\n",
      "Step 4 | Training Loss: 0.000356 | Train Accuracy: 1.000000 | Test Accuracy: 0.927475, 0.862025\n",
      "Step 5 | Training Loss: 0.000170 | Train Accuracy: 1.000000 | Test Accuracy: 0.931778, 0.870211\n",
      "Step 1 | Training Loss: 0.000120 | Train Accuracy: 1.000000 | Test Accuracy: 0.932887, 0.872321\n",
      "Step 2 | Training Loss: 0.000238 | Train Accuracy: 1.000000 | Test Accuracy: 0.933597, 0.873671\n",
      "Step 3 | Training Loss: 0.000072 | Train Accuracy: 1.000000 | Test Accuracy: 0.933863, 0.874177\n",
      "Step 4 | Training Loss: 0.000063 | Train Accuracy: 1.000000 | Test Accuracy: 0.933641, 0.873755\n",
      "Step 5 | Training Loss: 0.000041 | Train Accuracy: 1.000000 | Test Accuracy: 0.934839, 0.876034\n",
      "Step 1 | Training Loss: 0.000130 | Train Accuracy: 0.999921 | Test Accuracy: 0.934661, 0.875696\n",
      "Step 2 | Training Loss: 0.000033 | Train Accuracy: 1.000000 | Test Accuracy: 0.935593, 0.877468\n",
      "Step 3 | Training Loss: 0.000050 | Train Accuracy: 1.000000 | Test Accuracy: 0.938698, 0.883376\n",
      "Step 4 | Training Loss: 0.000055 | Train Accuracy: 1.000000 | Test Accuracy: 0.940738, 0.887257\n",
      "Step 5 | Training Loss: 0.000029 | Train Accuracy: 0.999921 | Test Accuracy: 0.942246, 0.890127\n",
      "Current Layer Attributes - epochs:5 hidden layers:3 features count:1\n",
      "Initial Accuracy, before training: 0.6971256136894226\n",
      "Step 1 | Training Loss: 0.001259 | Train Accuracy: 0.999841 | Test Accuracy: 0.979729, 0.961435\n",
      "Step 2 | Training Loss: 0.000001 | Train Accuracy: 0.999682 | Test Accuracy: 0.981725, 0.965232\n",
      "Step 3 | Training Loss: 0.000853 | Train Accuracy: 0.999921 | Test Accuracy: 0.983100, 0.967848\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.983410, 0.968439\n",
      "Step 5 | Training Loss: 0.000030 | Train Accuracy: 0.999921 | Test Accuracy: 0.999867, 0.999747\n",
      "Step 1 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999956, 0.999916\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999956, 0.999916\n",
      "Step 3 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999956, 0.999916\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999956, 0.999916\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999956, 0.999916\n",
      "Step 1 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999956, 0.999916\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999956, 0.999916\n",
      "Step 3 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999956, 0.999916\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999956, 0.999916\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999956, 0.999916\n",
      "Current Layer Attributes - epochs:5 hidden layers:5 features count:1\n",
      "Initial Accuracy, before training: 0.5984740853309631\n",
      "Step 1 | Training Loss: 0.014215 | Train Accuracy: 0.977933 | Test Accuracy: 0.800080, 0.621097\n",
      "Step 2 | Training Loss: 0.016396 | Train Accuracy: 0.984601 | Test Accuracy: 0.783046, 0.587257\n",
      "Step 3 | Training Loss: 0.000149 | Train Accuracy: 0.999127 | Test Accuracy: 0.900772, 0.811224\n",
      "Step 4 | Training Loss: 0.001780 | Train Accuracy: 0.998571 | Test Accuracy: 0.975603, 0.953587\n",
      "Step 5 | Training Loss: 0.000003 | Train Accuracy: 0.999444 | Test Accuracy: 0.975958, 0.954262\n",
      "Step 1 | Training Loss: 0.000002 | Train Accuracy: 0.999444 | Test Accuracy: 0.977866, 0.957890\n",
      "Step 2 | Training Loss: 0.000001 | Train Accuracy: 0.999682 | Test Accuracy: 0.978220, 0.958565\n",
      "Step 3 | Training Loss: 0.003521 | Train Accuracy: 0.999682 | Test Accuracy: 0.978220, 0.958565\n",
      "Step 4 | Training Loss: 0.000001 | Train Accuracy: 0.999603 | Test Accuracy: 0.978531, 0.959156\n",
      "Step 5 | Training Loss: 0.000001 | Train Accuracy: 0.999682 | Test Accuracy: 0.979773, 0.961519\n",
      "Step 1 | Training Loss: 0.000001 | Train Accuracy: 0.999365 | Test Accuracy: 0.981769, 0.965316\n",
      "Step 2 | Training Loss: 0.001761 | Train Accuracy: 0.999682 | Test Accuracy: 0.981902, 0.965570\n",
      "Step 3 | Training Loss: 0.001767 | Train Accuracy: 0.999444 | Test Accuracy: 0.981902, 0.965570\n",
      "Step 4 | Training Loss: 0.000001 | Train Accuracy: 0.999921 | Test Accuracy: 0.981946, 0.965654\n",
      "Step 5 | Training Loss: 0.001730 | Train Accuracy: 0.999841 | Test Accuracy: 0.981946, 0.965654\n",
      "Current Layer Attributes - epochs:5 hidden layers:1 features count:1\n",
      "Initial Accuracy, before training: 0.6804915070533752\n",
      "Step 1 | Training Loss: 0.001360 | Train Accuracy: 0.998809 | Test Accuracy: 0.831973, 0.680338\n",
      "Step 2 | Training Loss: 0.000259 | Train Accuracy: 0.999841 | Test Accuracy: 0.907115, 0.823291\n",
      "Step 3 | Training Loss: 0.000170 | Train Accuracy: 1.000000 | Test Accuracy: 0.930403, 0.867595\n",
      "Step 4 | Training Loss: 0.000098 | Train Accuracy: 1.000000 | Test Accuracy: 0.938343, 0.882700\n",
      "Step 5 | Training Loss: 0.000093 | Train Accuracy: 1.000000 | Test Accuracy: 0.934084, 0.874599\n",
      "Step 1 | Training Loss: 0.000034 | Train Accuracy: 1.000000 | Test Accuracy: 0.935992, 0.878228\n",
      "Step 2 | Training Loss: 0.000050 | Train Accuracy: 1.000000 | Test Accuracy: 0.937988, 0.882025\n",
      "Step 3 | Training Loss: 0.000038 | Train Accuracy: 1.000000 | Test Accuracy: 0.937234, 0.880591\n",
      "Step 4 | Training Loss: 0.000041 | Train Accuracy: 1.000000 | Test Accuracy: 0.955110, 0.914599\n",
      "Step 5 | Training Loss: 0.000040 | Train Accuracy: 1.000000 | Test Accuracy: 0.955420, 0.915190\n",
      "Step 1 | Training Loss: 0.000020 | Train Accuracy: 1.000000 | Test Accuracy: 0.955509, 0.915359\n",
      "Step 2 | Training Loss: 0.000019 | Train Accuracy: 1.000000 | Test Accuracy: 0.956130, 0.916540\n",
      "Step 3 | Training Loss: 0.000041 | Train Accuracy: 1.000000 | Test Accuracy: 0.956086, 0.916456\n",
      "Step 4 | Training Loss: 0.000011 | Train Accuracy: 1.000000 | Test Accuracy: 0.956396, 0.917046\n",
      "Step 5 | Training Loss: 0.000019 | Train Accuracy: 1.000000 | Test Accuracy: 0.956796, 0.917806\n",
      "Current Layer Attributes - epochs:5 hidden layers:3 features count:1\n",
      "Initial Accuracy, before training: 0.6314762234687805\n",
      "Step 1 | Training Loss: 0.000002 | Train Accuracy: 0.999524 | Test Accuracy: 0.997605, 0.995443\n",
      "Step 2 | Training Loss: 0.000001 | Train Accuracy: 0.999762 | Test Accuracy: 0.997560, 0.995359\n",
      "Step 3 | Training Loss: 0.000001 | Train Accuracy: 0.999127 | Test Accuracy: 0.982967, 0.967595\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.982346, 0.966413\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.982434, 0.966582\n",
      "Step 1 | Training Loss: 0.000000 | Train Accuracy: 0.999841 | Test Accuracy: 0.982567, 0.966835\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 0.999762 | Test Accuracy: 0.982567, 0.966835\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3 | Training Loss: 0.000894 | Train Accuracy: 0.999762 | Test Accuracy: 0.982612, 0.966920\n",
      "Step 4 | Training Loss: 0.000007 | Train Accuracy: 1.000000 | Test Accuracy: 0.982745, 0.967173\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.982745, 0.967173\n",
      "Step 1 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.982745, 0.967173\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.982745, 0.967173\n",
      "Step 3 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.982745, 0.967173\n",
      "Step 4 | Training Loss: 0.000003 | Train Accuracy: 1.000000 | Test Accuracy: 0.982745, 0.967173\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.982745, 0.967173\n",
      "Current Layer Attributes - epochs:5 hidden layers:5 features count:1\n",
      "Initial Accuracy, before training: 0.7603353261947632\n",
      "Step 1 | Training Loss: 0.004781 | Train Accuracy: 0.998492 | Test Accuracy: 0.973208, 0.949367\n",
      "Step 2 | Training Loss: 0.001764 | Train Accuracy: 0.999762 | Test Accuracy: 0.985672, 0.972743\n",
      "Step 3 | Training Loss: 0.001769 | Train Accuracy: 0.999444 | Test Accuracy: 0.998270, 0.996709\n",
      "Step 4 | Training Loss: 0.003466 | Train Accuracy: 0.998889 | Test Accuracy: 0.997738, 0.995696\n",
      "Step 5 | Training Loss: 0.000002 | Train Accuracy: 0.999841 | Test Accuracy: 0.996939, 0.994177\n",
      "Step 1 | Training Loss: 0.000001 | Train Accuracy: 0.999841 | Test Accuracy: 0.997560, 0.995359\n",
      "Step 2 | Training Loss: 0.000001 | Train Accuracy: 0.999762 | Test Accuracy: 0.997383, 0.995021\n",
      "Step 3 | Training Loss: 0.000001 | Train Accuracy: 0.999921 | Test Accuracy: 0.997826, 0.995865\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 0.999603 | Test Accuracy: 0.997826, 0.995865\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.997826, 0.995865\n",
      "Step 1 | Training Loss: 0.003532 | Train Accuracy: 0.999762 | Test Accuracy: 0.997826, 0.995865\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 0.999762 | Test Accuracy: 0.997826, 0.995865\n",
      "Step 3 | Training Loss: 0.000000 | Train Accuracy: 0.999841 | Test Accuracy: 0.997826, 0.995865\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 0.999762 | Test Accuracy: 0.997826, 0.995865\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 0.999762 | Test Accuracy: 0.997826, 0.995865\n",
      "Current Layer Attributes - epochs:5 hidden layers:1 features count:1\n",
      "Initial Accuracy, before training: 0.26969480514526367\n",
      "Step 1 | Training Loss: 0.006713 | Train Accuracy: 0.997777 | Test Accuracy: 0.803983, 0.627173\n",
      "Step 2 | Training Loss: 0.001863 | Train Accuracy: 0.999444 | Test Accuracy: 0.862136, 0.737722\n",
      "Step 3 | Training Loss: 0.000161 | Train Accuracy: 0.999682 | Test Accuracy: 0.899308, 0.808439\n",
      "Step 4 | Training Loss: 0.000312 | Train Accuracy: 0.999921 | Test Accuracy: 0.919003, 0.845907\n",
      "Step 5 | Training Loss: 0.000057 | Train Accuracy: 0.999921 | Test Accuracy: 0.939186, 0.884304\n",
      "Step 1 | Training Loss: 0.000145 | Train Accuracy: 1.000000 | Test Accuracy: 0.940871, 0.887511\n",
      "Step 2 | Training Loss: 0.000109 | Train Accuracy: 0.999841 | Test Accuracy: 0.941270, 0.888270\n",
      "Step 3 | Training Loss: 0.000028 | Train Accuracy: 0.999841 | Test Accuracy: 0.940916, 0.887595\n",
      "Step 4 | Training Loss: 0.000022 | Train Accuracy: 0.999921 | Test Accuracy: 0.942291, 0.890211\n",
      "Step 5 | Training Loss: 0.000036 | Train Accuracy: 1.000000 | Test Accuracy: 0.942202, 0.890042\n",
      "Step 1 | Training Loss: 0.000061 | Train Accuracy: 1.000000 | Test Accuracy: 0.942335, 0.890295\n",
      "Step 2 | Training Loss: 0.000039 | Train Accuracy: 0.999921 | Test Accuracy: 0.942557, 0.890717\n",
      "Step 3 | Training Loss: 0.000931 | Train Accuracy: 1.000000 | Test Accuracy: 0.942867, 0.891308\n",
      "Step 4 | Training Loss: 0.000026 | Train Accuracy: 0.999921 | Test Accuracy: 0.943488, 0.892489\n",
      "Step 5 | Training Loss: 0.000035 | Train Accuracy: 0.999921 | Test Accuracy: 0.943843, 0.893165\n",
      "Current Layer Attributes - epochs:5 hidden layers:3 features count:1\n",
      "Initial Accuracy, before training: 0.5897799730300903\n",
      "Step 1 | Training Loss: 0.001744 | Train Accuracy: 0.999286 | Test Accuracy: 0.982656, 0.967004\n",
      "Step 2 | Training Loss: 0.000240 | Train Accuracy: 0.999365 | Test Accuracy: 0.995298, 0.991055\n",
      "Step 3 | Training Loss: 0.000002 | Train Accuracy: 0.999762 | Test Accuracy: 0.983321, 0.968270\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 0.999365 | Test Accuracy: 0.998802, 0.997721\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 0.999682 | Test Accuracy: 0.998669, 0.997468\n",
      "Step 1 | Training Loss: 0.003535 | Train Accuracy: 0.999524 | Test Accuracy: 0.998669, 0.997468\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 0.999286 | Test Accuracy: 0.998669, 0.997468\n",
      "Step 3 | Training Loss: 0.000000 | Train Accuracy: 0.999762 | Test Accuracy: 0.998714, 0.997553\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 0.999762 | Test Accuracy: 0.998714, 0.997553\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 0.999603 | Test Accuracy: 0.998758, 0.997637\n",
      "Step 1 | Training Loss: 0.001767 | Train Accuracy: 0.999603 | Test Accuracy: 0.998758, 0.997637\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 0.999603 | Test Accuracy: 0.998758, 0.997637\n",
      "Step 3 | Training Loss: 0.000000 | Train Accuracy: 0.999603 | Test Accuracy: 0.998758, 0.997637\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 0.999682 | Test Accuracy: 0.998758, 0.997637\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 0.999603 | Test Accuracy: 0.998847, 0.997806\n",
      "Current Layer Attributes - epochs:5 hidden layers:5 features count:1\n",
      "Initial Accuracy, before training: 0.5375709533691406\n",
      "Step 1 | Training Loss: 0.028795 | Train Accuracy: 0.975869 | Test Accuracy: 0.764017, 0.552236\n",
      "Step 2 | Training Loss: 0.017046 | Train Accuracy: 0.985236 | Test Accuracy: 0.757940, 0.540675\n",
      "Step 3 | Training Loss: 0.007034 | Train Accuracy: 0.991983 | Test Accuracy: 0.771159, 0.566160\n",
      "Step 4 | Training Loss: 0.004413 | Train Accuracy: 0.990475 | Test Accuracy: 0.770848, 0.564219\n",
      "Step 5 | Training Loss: 0.004568 | Train Accuracy: 0.990951 | Test Accuracy: 0.773909, 0.570127\n",
      "Step 1 | Training Loss: 0.004503 | Train Accuracy: 0.993491 | Test Accuracy: 0.778167, 0.578059\n",
      "Step 2 | Training Loss: 0.000443 | Train Accuracy: 0.994285 | Test Accuracy: 0.772800, 0.568186\n",
      "Step 3 | Training Loss: 0.004171 | Train Accuracy: 0.994285 | Test Accuracy: 0.769384, 0.561350\n",
      "Step 4 | Training Loss: 0.003424 | Train Accuracy: 0.994761 | Test Accuracy: 0.773953, 0.570127\n",
      "Step 5 | Training Loss: 0.000211 | Train Accuracy: 0.992697 | Test Accuracy: 0.781627, 0.585232\n",
      "Step 1 | Training Loss: 0.002419 | Train Accuracy: 0.996349 | Test Accuracy: 0.778389, 0.578903\n",
      "Step 2 | Training Loss: 0.001653 | Train Accuracy: 0.996110 | Test Accuracy: 0.779498, 0.580675\n",
      "Step 3 | Training Loss: 0.002940 | Train Accuracy: 0.996349 | Test Accuracy: 0.777768, 0.577637\n",
      "Step 4 | Training Loss: 0.004739 | Train Accuracy: 0.996110 | Test Accuracy: 0.776659, 0.575612\n",
      "Step 5 | Training Loss: 0.000518 | Train Accuracy: 0.995793 | Test Accuracy: 0.780429, 0.582700\n",
      "Current Layer Attributes - epochs:5 hidden layers:1 features count:1\n",
      "Initial Accuracy, before training: 0.3840489685535431\n",
      "Step 1 | Training Loss: 0.003893 | Train Accuracy: 0.998333 | Test Accuracy: 0.828779, 0.674262\n",
      "Step 2 | Training Loss: 0.001749 | Train Accuracy: 0.999524 | Test Accuracy: 0.879524, 0.770802\n",
      "Step 3 | Training Loss: 0.000540 | Train Accuracy: 0.999762 | Test Accuracy: 0.906095, 0.821350\n",
      "Step 4 | Training Loss: 0.000145 | Train Accuracy: 1.000000 | Test Accuracy: 0.928584, 0.864135\n",
      "Step 5 | Training Loss: 0.000133 | Train Accuracy: 0.999841 | Test Accuracy: 0.933330, 0.873165\n",
      "Step 1 | Training Loss: 0.000101 | Train Accuracy: 1.000000 | Test Accuracy: 0.936879, 0.879916\n",
      "Step 2 | Training Loss: 0.000066 | Train Accuracy: 1.000000 | Test Accuracy: 0.937810, 0.881688\n",
      "Step 3 | Training Loss: 0.000233 | Train Accuracy: 1.000000 | Test Accuracy: 0.938653, 0.883291\n",
      "Step 4 | Training Loss: 0.000074 | Train Accuracy: 1.000000 | Test Accuracy: 0.939407, 0.884726\n",
      "Step 5 | Training Loss: 0.000091 | Train Accuracy: 1.000000 | Test Accuracy: 0.939984, 0.885823\n",
      "Step 1 | Training Loss: 0.000059 | Train Accuracy: 1.000000 | Test Accuracy: 0.940339, 0.886498\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2 | Training Loss: 0.000039 | Train Accuracy: 1.000000 | Test Accuracy: 0.940605, 0.887004\n",
      "Step 3 | Training Loss: 0.000053 | Train Accuracy: 1.000000 | Test Accuracy: 0.940605, 0.887004\n",
      "Step 4 | Training Loss: 0.000035 | Train Accuracy: 1.000000 | Test Accuracy: 0.940960, 0.887679\n",
      "Step 5 | Training Loss: 0.000049 | Train Accuracy: 1.000000 | Test Accuracy: 0.941670, 0.889030\n",
      "Current Layer Attributes - epochs:5 hidden layers:3 features count:1\n",
      "Initial Accuracy, before training: 0.5593506097793579\n",
      "Step 1 | Training Loss: 0.000022 | Train Accuracy: 0.999603 | Test Accuracy: 0.997560, 0.995359\n",
      "Step 2 | Training Loss: 0.000004 | Train Accuracy: 0.999603 | Test Accuracy: 0.996584, 0.993502\n",
      "Step 3 | Training Loss: 0.001769 | Train Accuracy: 0.999762 | Test Accuracy: 0.997826, 0.995865\n",
      "Step 4 | Training Loss: 0.000001 | Train Accuracy: 0.999762 | Test Accuracy: 0.997826, 0.995865\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.997782, 0.995781\n",
      "Step 1 | Training Loss: 0.000000 | Train Accuracy: 0.999762 | Test Accuracy: 0.997826, 0.995865\n",
      "Step 2 | Training Loss: 0.000002 | Train Accuracy: 0.999762 | Test Accuracy: 0.997826, 0.995865\n",
      "Step 3 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.997826, 0.995865\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 0.999682 | Test Accuracy: 0.997826, 0.995865\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.997826, 0.995865\n",
      "Step 1 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.997826, 0.995865\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 0.999762 | Test Accuracy: 0.997826, 0.995865\n",
      "Step 3 | Training Loss: 0.000000 | Train Accuracy: 0.999524 | Test Accuracy: 0.997826, 0.995865\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.997871, 0.995949\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.997560, 0.995359\n",
      "Current Layer Attributes - epochs:5 hidden layers:5 features count:1\n",
      "Initial Accuracy, before training: 0.6384847164154053\n",
      "Step 1 | Training Loss: 0.029366 | Train Accuracy: 0.973885 | Test Accuracy: 0.745387, 0.517131\n",
      "Step 2 | Training Loss: 0.029334 | Train Accuracy: 0.971742 | Test Accuracy: 0.754968, 0.536540\n",
      "Step 3 | Training Loss: 0.011454 | Train Accuracy: 0.981743 | Test Accuracy: 0.753549, 0.533418\n",
      "Step 4 | Training Loss: 0.012727 | Train Accuracy: 0.984918 | Test Accuracy: 0.747250, 0.519325\n",
      "Step 5 | Training Loss: 0.006105 | Train Accuracy: 0.994364 | Test Accuracy: 0.784422, 0.589873\n",
      "Step 1 | Training Loss: 0.006410 | Train Accuracy: 0.994682 | Test Accuracy: 0.761045, 0.545485\n",
      "Step 2 | Training Loss: 0.000950 | Train Accuracy: 0.997777 | Test Accuracy: 0.797152, 0.614093\n",
      "Step 3 | Training Loss: 0.001512 | Train Accuracy: 0.998968 | Test Accuracy: 0.800035, 0.619578\n",
      "Step 4 | Training Loss: 0.003324 | Train Accuracy: 0.999286 | Test Accuracy: 0.833703, 0.683629\n",
      "Step 5 | Training Loss: 0.001100 | Train Accuracy: 0.999127 | Test Accuracy: 0.856946, 0.727848\n",
      "Step 1 | Training Loss: 0.000005 | Train Accuracy: 0.999682 | Test Accuracy: 0.938121, 0.882279\n",
      "Step 2 | Training Loss: 0.000017 | Train Accuracy: 0.999444 | Test Accuracy: 0.969349, 0.941688\n",
      "Step 3 | Training Loss: 0.000013 | Train Accuracy: 0.999603 | Test Accuracy: 0.968772, 0.940591\n",
      "Step 4 | Training Loss: 0.000007 | Train Accuracy: 0.999841 | Test Accuracy: 0.937456, 0.881013\n",
      "Step 5 | Training Loss: 0.000008 | Train Accuracy: 0.999682 | Test Accuracy: 0.968905, 0.940844\n",
      "Current Layer Attributes - epochs:5 hidden layers:1 features count:1\n",
      "Initial Accuracy, before training: 0.588449239730835\n",
      "Step 1 | Training Loss: 0.006069 | Train Accuracy: 0.990872 | Test Accuracy: 0.798838, 0.617975\n",
      "Step 2 | Training Loss: 0.001144 | Train Accuracy: 0.999841 | Test Accuracy: 0.878238, 0.768354\n",
      "Step 3 | Training Loss: 0.000193 | Train Accuracy: 0.999762 | Test Accuracy: 0.937367, 0.880844\n",
      "Step 4 | Training Loss: 0.000919 | Train Accuracy: 1.000000 | Test Accuracy: 0.982168, 0.966076\n",
      "Step 5 | Training Loss: 0.000042 | Train Accuracy: 1.000000 | Test Accuracy: 0.970059, 0.943038\n",
      "Step 1 | Training Loss: 0.000035 | Train Accuracy: 1.000000 | Test Accuracy: 0.979995, 0.961941\n",
      "Step 2 | Training Loss: 0.000042 | Train Accuracy: 1.000000 | Test Accuracy: 0.967175, 0.937553\n",
      "Step 3 | Training Loss: 0.000023 | Train Accuracy: 1.000000 | Test Accuracy: 0.962074, 0.927848\n",
      "Step 4 | Training Loss: 0.000019 | Train Accuracy: 1.000000 | Test Accuracy: 0.960477, 0.924810\n",
      "Step 5 | Training Loss: 0.000026 | Train Accuracy: 1.000000 | Test Accuracy: 0.961364, 0.926498\n",
      "Step 1 | Training Loss: 0.000022 | Train Accuracy: 1.000000 | Test Accuracy: 0.960743, 0.925316\n",
      "Step 2 | Training Loss: 0.000023 | Train Accuracy: 1.000000 | Test Accuracy: 0.959945, 0.923797\n",
      "Step 3 | Training Loss: 0.000010 | Train Accuracy: 1.000000 | Test Accuracy: 0.959147, 0.922278\n",
      "Step 4 | Training Loss: 0.000034 | Train Accuracy: 1.000000 | Test Accuracy: 0.959324, 0.922616\n",
      "Step 5 | Training Loss: 0.000027 | Train Accuracy: 1.000000 | Test Accuracy: 0.959280, 0.922532\n",
      "Current Layer Attributes - epochs:5 hidden layers:3 features count:1\n",
      "Initial Accuracy, before training: 0.372826486825943\n",
      "Step 1 | Training Loss: 0.000003 | Train Accuracy: 0.999047 | Test Accuracy: 0.999335, 0.998734\n",
      "Step 2 | Training Loss: 0.000001 | Train Accuracy: 0.999762 | Test Accuracy: 0.997738, 0.995696\n",
      "Step 3 | Training Loss: 0.000002 | Train Accuracy: 0.999762 | Test Accuracy: 0.997250, 0.994768\n",
      "Step 4 | Training Loss: 0.000003 | Train Accuracy: 1.000000 | Test Accuracy: 0.997738, 0.995696\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.997294, 0.994852\n",
      "Step 1 | Training Loss: 0.000003 | Train Accuracy: 0.999921 | Test Accuracy: 0.997294, 0.994852\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.997294, 0.994852\n",
      "Step 3 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.997294, 0.994852\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.998004, 0.996203\n",
      "Step 5 | Training Loss: 0.000001 | Train Accuracy: 1.000000 | Test Accuracy: 0.997960, 0.996118\n",
      "Step 1 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.997960, 0.996118\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.997915, 0.996034\n",
      "Step 3 | Training Loss: 0.000001 | Train Accuracy: 1.000000 | Test Accuracy: 0.997915, 0.996034\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.997915, 0.996034\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.997915, 0.996034\n",
      "Current Layer Attributes - epochs:5 hidden layers:5 features count:1\n",
      "Initial Accuracy, before training: 0.4764017164707184\n",
      "Step 1 | Training Loss: 0.023943 | Train Accuracy: 0.965947 | Test Accuracy: 0.785841, 0.594515\n",
      "Step 2 | Training Loss: 0.006488 | Train Accuracy: 0.986585 | Test Accuracy: 0.852156, 0.720338\n",
      "Step 3 | Training Loss: 0.002552 | Train Accuracy: 0.987935 | Test Accuracy: 0.821771, 0.662110\n",
      "Step 4 | Training Loss: 0.000031 | Train Accuracy: 0.999841 | Test Accuracy: 0.969349, 0.941688\n",
      "Step 5 | Training Loss: 0.000004 | Train Accuracy: 0.999365 | Test Accuracy: 0.981148, 0.964135\n",
      "Step 1 | Training Loss: 0.000009 | Train Accuracy: 0.999841 | Test Accuracy: 0.987314, 0.975865\n",
      "Step 2 | Training Loss: 0.000005 | Train Accuracy: 0.999841 | Test Accuracy: 0.989132, 0.979325\n",
      "Step 3 | Training Loss: 0.000004 | Train Accuracy: 0.999921 | Test Accuracy: 0.989132, 0.979325\n",
      "Step 4 | Training Loss: 0.000002 | Train Accuracy: 1.000000 | Test Accuracy: 0.987358, 0.975949\n",
      "Step 5 | Training Loss: 0.000001 | Train Accuracy: 0.999841 | Test Accuracy: 0.986648, 0.974599\n",
      "Step 1 | Training Loss: 0.000000 | Train Accuracy: 0.999841 | Test Accuracy: 0.986737, 0.974768\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.986870, 0.975021\n",
      "Step 3 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.987181, 0.975612\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.987491, 0.976203\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.987713, 0.976624\n",
      "Current Layer Attributes - epochs:5 hidden layers:1 features count:1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initial Accuracy, before training: 0.6875443458557129\n",
      "Step 1 | Training Loss: 0.002989 | Train Accuracy: 0.999286 | Test Accuracy: 0.836764, 0.689451\n",
      "Step 2 | Training Loss: 0.000408 | Train Accuracy: 1.000000 | Test Accuracy: 0.917938, 0.843882\n",
      "Step 3 | Training Loss: 0.000255 | Train Accuracy: 1.000000 | Test Accuracy: 0.929870, 0.866582\n",
      "Step 4 | Training Loss: 0.001016 | Train Accuracy: 1.000000 | Test Accuracy: 0.931112, 0.868945\n",
      "Step 5 | Training Loss: 0.000094 | Train Accuracy: 1.000000 | Test Accuracy: 0.956175, 0.916624\n",
      "Step 1 | Training Loss: 0.000048 | Train Accuracy: 1.000000 | Test Accuracy: 0.938831, 0.883629\n",
      "Step 2 | Training Loss: 0.000095 | Train Accuracy: 0.999921 | Test Accuracy: 0.939762, 0.885401\n",
      "Step 3 | Training Loss: 0.000069 | Train Accuracy: 1.000000 | Test Accuracy: 0.939629, 0.885148\n",
      "Step 4 | Training Loss: 0.000024 | Train Accuracy: 0.999841 | Test Accuracy: 0.940428, 0.886667\n",
      "Step 5 | Training Loss: 0.000047 | Train Accuracy: 0.999921 | Test Accuracy: 0.940827, 0.887426\n",
      "Step 1 | Training Loss: 0.000023 | Train Accuracy: 0.999841 | Test Accuracy: 0.941492, 0.888692\n",
      "Step 2 | Training Loss: 0.000029 | Train Accuracy: 1.000000 | Test Accuracy: 0.941847, 0.889367\n",
      "Step 3 | Training Loss: 0.000040 | Train Accuracy: 1.000000 | Test Accuracy: 0.941492, 0.888692\n",
      "Step 4 | Training Loss: 0.000017 | Train Accuracy: 1.000000 | Test Accuracy: 0.941448, 0.888608\n",
      "Step 5 | Training Loss: 0.000057 | Train Accuracy: 0.999921 | Test Accuracy: 0.941891, 0.889452\n",
      "Current Layer Attributes - epochs:5 hidden layers:3 features count:1\n",
      "Initial Accuracy, before training: 0.33223918080329895\n",
      "Step 1 | Training Loss: 0.000001 | Train Accuracy: 0.999444 | Test Accuracy: 0.998847, 0.997806\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 0.999841 | Test Accuracy: 0.999113, 0.998312\n",
      "Step 3 | Training Loss: 0.001419 | Train Accuracy: 1.000000 | Test Accuracy: 0.999468, 0.998987\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999423, 0.998903\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999423, 0.998903\n",
      "Step 1 | Training Loss: 0.000000 | Train Accuracy: 0.999841 | Test Accuracy: 0.999423, 0.998903\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999423, 0.998903\n",
      "Step 3 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999423, 0.998903\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.999468, 0.998987\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999468, 0.998987\n",
      "Step 1 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999468, 0.998987\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999468, 0.998987\n",
      "Step 3 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999468, 0.998987\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999468, 0.998987\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.999468, 0.998987\n",
      "Current Layer Attributes - epochs:5 hidden layers:5 features count:1\n",
      "Initial Accuracy, before training: 0.6654542088508606\n",
      "Step 1 | Training Loss: 0.012044 | Train Accuracy: 0.973170 | Test Accuracy: 0.737136, 0.501266\n",
      "Step 2 | Training Loss: 0.019624 | Train Accuracy: 0.979759 | Test Accuracy: 0.807399, 0.635105\n",
      "Step 3 | Training Loss: 0.020175 | Train Accuracy: 0.983093 | Test Accuracy: 0.786551, 0.595443\n",
      "Step 4 | Training Loss: 0.004278 | Train Accuracy: 0.987617 | Test Accuracy: 0.794047, 0.609705\n",
      "Step 5 | Training Loss: 0.007113 | Train Accuracy: 0.991665 | Test Accuracy: 0.775195, 0.574177\n",
      "Step 1 | Training Loss: 0.006551 | Train Accuracy: 0.990475 | Test Accuracy: 0.774663, 0.573165\n",
      "Step 2 | Training Loss: 0.004136 | Train Accuracy: 0.991983 | Test Accuracy: 0.775594, 0.574599\n",
      "Step 3 | Training Loss: 0.007239 | Train Accuracy: 0.993094 | Test Accuracy: 0.794047, 0.609705\n",
      "Step 4 | Training Loss: 0.004331 | Train Accuracy: 0.995396 | Test Accuracy: 0.792229, 0.606245\n",
      "Step 5 | Training Loss: 0.004040 | Train Accuracy: 0.997936 | Test Accuracy: 0.907825, 0.826160\n",
      "Step 1 | Training Loss: 0.000007 | Train Accuracy: 0.999286 | Test Accuracy: 0.940871, 0.887595\n",
      "Step 2 | Training Loss: 0.000025 | Train Accuracy: 0.999841 | Test Accuracy: 0.961276, 0.926329\n",
      "Step 3 | Training Loss: 0.002310 | Train Accuracy: 0.999444 | Test Accuracy: 0.962961, 0.929620\n",
      "Step 4 | Training Loss: 0.000041 | Train Accuracy: 0.999682 | Test Accuracy: 0.964824, 0.933165\n",
      "Step 5 | Training Loss: 0.000004 | Train Accuracy: 0.999762 | Test Accuracy: 0.965401, 0.934262\n",
      "Current Layer Attributes - epochs:5 hidden layers:1 features count:1\n",
      "Initial Accuracy, before training: 0.45475515723228455\n",
      "Step 1 | Training Loss: 0.000396 | Train Accuracy: 0.999524 | Test Accuracy: 0.896913, 0.803882\n",
      "Step 2 | Training Loss: 0.000057 | Train Accuracy: 0.999921 | Test Accuracy: 0.947303, 0.899747\n",
      "Step 3 | Training Loss: 0.000069 | Train Accuracy: 1.000000 | Test Accuracy: 0.944331, 0.894093\n",
      "Step 4 | Training Loss: 0.000060 | Train Accuracy: 0.999841 | Test Accuracy: 0.944642, 0.894684\n",
      "Step 5 | Training Loss: 0.000015 | Train Accuracy: 0.999841 | Test Accuracy: 0.944819, 0.895021\n",
      "Step 1 | Training Loss: 0.000014 | Train Accuracy: 0.999921 | Test Accuracy: 0.944863, 0.895105\n",
      "Step 2 | Training Loss: 0.000011 | Train Accuracy: 1.000000 | Test Accuracy: 0.945928, 0.897131\n",
      "Step 3 | Training Loss: 0.000008 | Train Accuracy: 1.000000 | Test Accuracy: 0.945972, 0.897215\n",
      "Step 4 | Training Loss: 0.000021 | Train Accuracy: 1.000000 | Test Accuracy: 0.946904, 0.898987\n",
      "Step 5 | Training Loss: 0.000907 | Train Accuracy: 0.999841 | Test Accuracy: 0.947081, 0.899325\n",
      "Step 1 | Training Loss: 0.000004 | Train Accuracy: 1.000000 | Test Accuracy: 0.948101, 0.901266\n",
      "Step 2 | Training Loss: 0.000005 | Train Accuracy: 0.999841 | Test Accuracy: 0.948057, 0.901181\n",
      "Step 3 | Training Loss: 0.000005 | Train Accuracy: 0.999841 | Test Accuracy: 0.949388, 0.903713\n",
      "Step 4 | Training Loss: 0.000005 | Train Accuracy: 1.000000 | Test Accuracy: 0.949122, 0.903207\n",
      "Step 5 | Training Loss: 0.000006 | Train Accuracy: 1.000000 | Test Accuracy: 0.950231, 0.905316\n",
      "Current Layer Attributes - epochs:5 hidden layers:3 features count:1\n",
      "Initial Accuracy, before training: 0.7089691162109375\n",
      "Step 1 | Training Loss: 0.000002 | Train Accuracy: 0.999603 | Test Accuracy: 0.981503, 0.964810\n",
      "Step 2 | Training Loss: 0.000002 | Train Accuracy: 0.999682 | Test Accuracy: 0.981458, 0.964726\n",
      "Step 3 | Training Loss: 0.001645 | Train Accuracy: 0.999682 | Test Accuracy: 0.982612, 0.966920\n",
      "Step 4 | Training Loss: 0.000008 | Train Accuracy: 0.999603 | Test Accuracy: 0.986027, 0.973418\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 0.999682 | Test Accuracy: 0.982789, 0.967257\n",
      "Step 1 | Training Loss: 0.000000 | Train Accuracy: 0.999841 | Test Accuracy: 0.985185, 0.971814\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 0.999762 | Test Accuracy: 0.985495, 0.972405\n",
      "Step 3 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.994633, 0.989789\n",
      "Step 4 | Training Loss: 0.000001 | Train Accuracy: 1.000000 | Test Accuracy: 0.997782, 0.995781\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 0.999841 | Test Accuracy: 0.996451, 0.993249\n",
      "Step 1 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.997782, 0.995781\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.997782, 0.995781\n",
      "Step 3 | Training Loss: 0.000000 | Train Accuracy: 0.999841 | Test Accuracy: 0.997782, 0.995781\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.997782, 0.995781\n",
      "Step 5 | Training Loss: 0.000002 | Train Accuracy: 0.999841 | Test Accuracy: 0.985140, 0.971730\n",
      "Current Layer Attributes - epochs:5 hidden layers:5 features count:1\n",
      "Initial Accuracy, before training: 0.4045865833759308\n",
      "Step 1 | Training Loss: 0.025833 | Train Accuracy: 0.950865 | Test Accuracy: 0.765703, 0.557553\n",
      "Step 2 | Training Loss: 0.014012 | Train Accuracy: 0.986506 | Test Accuracy: 0.744189, 0.514515\n",
      "Step 3 | Training Loss: 0.007475 | Train Accuracy: 0.991268 | Test Accuracy: 0.779897, 0.582363\n",
      "Step 4 | Training Loss: 0.007330 | Train Accuracy: 0.991507 | Test Accuracy: 0.774885, 0.572405\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 5 | Training Loss: 0.005704 | Train Accuracy: 0.993332 | Test Accuracy: 0.786684, 0.594262\n",
      "Step 1 | Training Loss: 0.004059 | Train Accuracy: 0.994840 | Test Accuracy: 0.796576, 0.613165\n",
      "Step 2 | Training Loss: 0.007358 | Train Accuracy: 0.994761 | Test Accuracy: 0.787216, 0.595443\n",
      "Step 3 | Training Loss: 0.005621 | Train Accuracy: 0.994920 | Test Accuracy: 0.833925, 0.684473\n",
      "Step 4 | Training Loss: 0.006635 | Train Accuracy: 0.995555 | Test Accuracy: 0.784865, 0.590886\n",
      "Step 5 | Training Loss: 0.005466 | Train Accuracy: 0.994761 | Test Accuracy: 0.777457, 0.577384\n",
      "Step 1 | Training Loss: 0.003807 | Train Accuracy: 0.996031 | Test Accuracy: 0.786773, 0.594684\n",
      "Step 2 | Training Loss: 0.001771 | Train Accuracy: 0.996825 | Test Accuracy: 0.788946, 0.598565\n",
      "Step 3 | Training Loss: 0.002613 | Train Accuracy: 0.997301 | Test Accuracy: 0.810060, 0.639072\n",
      "Step 4 | Training Loss: 0.001716 | Train Accuracy: 0.996746 | Test Accuracy: 0.847188, 0.709451\n",
      "Step 5 | Training Loss: 0.001673 | Train Accuracy: 0.998571 | Test Accuracy: 0.921265, 0.850464\n",
      "Current Layer Attributes - epochs:5 hidden layers:1 features count:1\n",
      "Initial Accuracy, before training: 0.825141966342926\n",
      "Step 1 | Training Loss: 0.005486 | Train Accuracy: 0.997539 | Test Accuracy: 0.808863, 0.636371\n",
      "Step 2 | Training Loss: 0.001203 | Train Accuracy: 0.999762 | Test Accuracy: 0.881875, 0.775274\n",
      "Step 3 | Training Loss: 0.000215 | Train Accuracy: 0.999603 | Test Accuracy: 0.918825, 0.845570\n",
      "Step 4 | Training Loss: 0.000995 | Train Accuracy: 0.999682 | Test Accuracy: 0.956086, 0.916456\n",
      "Step 5 | Training Loss: 0.000124 | Train Accuracy: 0.999206 | Test Accuracy: 0.958259, 0.920591\n",
      "Step 1 | Training Loss: 0.000038 | Train Accuracy: 0.999921 | Test Accuracy: 0.958171, 0.920422\n",
      "Step 2 | Training Loss: 0.000058 | Train Accuracy: 0.999682 | Test Accuracy: 0.954888, 0.914177\n",
      "Step 3 | Training Loss: 0.000781 | Train Accuracy: 0.999841 | Test Accuracy: 0.957195, 0.918565\n",
      "Step 4 | Training Loss: 0.000049 | Train Accuracy: 0.999682 | Test Accuracy: 0.955687, 0.915696\n",
      "Step 5 | Training Loss: 0.000042 | Train Accuracy: 0.999365 | Test Accuracy: 0.957195, 0.918565\n",
      "Step 1 | Training Loss: 0.000898 | Train Accuracy: 0.999921 | Test Accuracy: 0.957372, 0.918903\n",
      "Step 2 | Training Loss: 0.000079 | Train Accuracy: 0.999921 | Test Accuracy: 0.957284, 0.918734\n",
      "Step 3 | Training Loss: 0.000057 | Train Accuracy: 0.999921 | Test Accuracy: 0.955066, 0.914515\n",
      "Step 4 | Training Loss: 0.000020 | Train Accuracy: 0.999921 | Test Accuracy: 0.956973, 0.918143\n",
      "Step 5 | Training Loss: 0.000017 | Train Accuracy: 0.999921 | Test Accuracy: 0.954356, 0.913165\n",
      "Current Layer Attributes - epochs:5 hidden layers:3 features count:1\n",
      "Initial Accuracy, before training: 0.35122427344322205\n",
      "Step 1 | Training Loss: 0.000004 | Train Accuracy: 1.000000 | Test Accuracy: 0.965224, 0.933840\n",
      "Step 2 | Training Loss: 0.000001 | Train Accuracy: 0.999762 | Test Accuracy: 0.998714, 0.997553\n",
      "Step 3 | Training Loss: 0.000001 | Train Accuracy: 1.000000 | Test Accuracy: 0.999290, 0.998650\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.999290, 0.998650\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999290, 0.998650\n",
      "Step 1 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.999290, 0.998650\n",
      "Step 2 | Training Loss: 0.000001 | Train Accuracy: 0.999921 | Test Accuracy: 0.999290, 0.998650\n",
      "Step 3 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999290, 0.998650\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.999290, 0.998650\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999290, 0.998650\n",
      "Step 1 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999290, 0.998650\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999290, 0.998650\n",
      "Step 3 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.999290, 0.998650\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999290, 0.998650\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.999290, 0.998650\n",
      "Current Layer Attributes - epochs:5 hidden layers:5 features count:1\n",
      "Initial Accuracy, before training: 0.6894517540931702\n",
      "Step 1 | Training Loss: 0.024689 | Train Accuracy: 0.974361 | Test Accuracy: 0.780829, 0.584895\n",
      "Step 2 | Training Loss: 0.022241 | Train Accuracy: 0.976425 | Test Accuracy: 0.738778, 0.504219\n",
      "Step 3 | Training Loss: 0.014032 | Train Accuracy: 0.985950 | Test Accuracy: 0.764949, 0.554008\n",
      "Step 4 | Training Loss: 0.007663 | Train Accuracy: 0.990872 | Test Accuracy: 0.754258, 0.533671\n",
      "Step 5 | Training Loss: 0.005187 | Train Accuracy: 0.989046 | Test Accuracy: 0.770981, 0.566076\n",
      "Step 1 | Training Loss: 0.003864 | Train Accuracy: 0.993015 | Test Accuracy: 0.782514, 0.587342\n",
      "Step 2 | Training Loss: 0.003295 | Train Accuracy: 0.995317 | Test Accuracy: 0.794402, 0.608861\n",
      "Step 3 | Training Loss: 0.005155 | Train Accuracy: 0.993015 | Test Accuracy: 0.788325, 0.597300\n",
      "Step 4 | Training Loss: 0.010982 | Train Accuracy: 0.994602 | Test Accuracy: 0.782780, 0.586835\n",
      "Step 5 | Training Loss: 0.003601 | Train Accuracy: 0.995634 | Test Accuracy: 0.797241, 0.614346\n",
      "Step 1 | Training Loss: 0.005937 | Train Accuracy: 0.997460 | Test Accuracy: 0.803229, 0.625738\n",
      "Step 2 | Training Loss: 0.000136 | Train Accuracy: 0.998651 | Test Accuracy: 0.951783, 0.908270\n",
      "Step 3 | Training Loss: 0.000866 | Train Accuracy: 0.999047 | Test Accuracy: 0.953203, 0.910970\n",
      "Step 4 | Training Loss: 0.001847 | Train Accuracy: 0.999921 | Test Accuracy: 0.954267, 0.912996\n",
      "Step 5 | Training Loss: 0.000011 | Train Accuracy: 0.999762 | Test Accuracy: 0.966288, 0.935865\n",
      "Current Layer Attributes - epochs:5 hidden layers:1 features count:1\n",
      "Initial Accuracy, before training: 0.25385913252830505\n",
      "Step 1 | Training Loss: 0.010233 | Train Accuracy: 0.989284 | Test Accuracy: 0.781317, 0.583966\n",
      "Step 2 | Training Loss: 0.002050 | Train Accuracy: 0.999206 | Test Accuracy: 0.840756, 0.697131\n",
      "Step 3 | Training Loss: 0.000738 | Train Accuracy: 0.999682 | Test Accuracy: 0.901437, 0.812489\n",
      "Step 4 | Training Loss: 0.000229 | Train Accuracy: 0.999762 | Test Accuracy: 0.926677, 0.860506\n",
      "Step 5 | Training Loss: 0.000237 | Train Accuracy: 1.000000 | Test Accuracy: 0.928007, 0.863038\n",
      "Step 1 | Training Loss: 0.000169 | Train Accuracy: 1.000000 | Test Accuracy: 0.927298, 0.861688\n",
      "Step 2 | Training Loss: 0.000070 | Train Accuracy: 0.999921 | Test Accuracy: 0.933242, 0.872996\n",
      "Step 3 | Training Loss: 0.000071 | Train Accuracy: 0.999841 | Test Accuracy: 0.931290, 0.869283\n",
      "Step 4 | Training Loss: 0.000154 | Train Accuracy: 1.000000 | Test Accuracy: 0.930004, 0.866835\n",
      "Step 5 | Training Loss: 0.000058 | Train Accuracy: 0.999841 | Test Accuracy: 0.933907, 0.874262\n",
      "Step 1 | Training Loss: 0.000043 | Train Accuracy: 0.999921 | Test Accuracy: 0.931157, 0.869030\n",
      "Step 2 | Training Loss: 0.000037 | Train Accuracy: 1.000000 | Test Accuracy: 0.931778, 0.870211\n",
      "Step 3 | Training Loss: 0.000092 | Train Accuracy: 0.999921 | Test Accuracy: 0.932887, 0.872321\n",
      "Step 4 | Training Loss: 0.000197 | Train Accuracy: 1.000000 | Test Accuracy: 0.930758, 0.868270\n",
      "Step 5 | Training Loss: 0.000107 | Train Accuracy: 0.999921 | Test Accuracy: 0.933375, 0.873249\n",
      "Current Layer Attributes - epochs:5 hidden layers:3 features count:1\n",
      "Initial Accuracy, before training: 0.7918736934661865\n",
      "Step 1 | Training Loss: 0.000009 | Train Accuracy: 0.999682 | Test Accuracy: 0.981148, 0.964135\n",
      "Step 2 | Training Loss: 0.000003 | Train Accuracy: 0.999841 | Test Accuracy: 0.998802, 0.997721\n",
      "Step 3 | Training Loss: 0.000001 | Train Accuracy: 1.000000 | Test Accuracy: 0.997782, 0.995781\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999024, 0.998143\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999468, 0.998987\n",
      "Step 1 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.999468, 0.998987\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999468, 0.998987\n",
      "Step 3 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999468, 0.998987\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999468, 0.998987\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999468, 0.998987\n",
      "Step 1 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999512, 0.999072\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999512, 0.999072\n",
      "Step 3 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999512, 0.999072\n",
      "Step 4 | Training Loss: 0.000001 | Train Accuracy: 1.000000 | Test Accuracy: 0.999512, 0.999072\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999512, 0.999072\n",
      "Current Layer Attributes - epochs:5 hidden layers:5 features count:1\n",
      "Initial Accuracy, before training: 0.7087029814720154\n",
      "Step 1 | Training Loss: 0.029560 | Train Accuracy: 0.982537 | Test Accuracy: 0.783801, 0.589789\n",
      "Step 2 | Training Loss: 0.000902 | Train Accuracy: 0.998968 | Test Accuracy: 0.979817, 0.962869\n",
      "Step 3 | Training Loss: 0.000316 | Train Accuracy: 0.999047 | Test Accuracy: 0.994455, 0.990717\n",
      "Step 4 | Training Loss: 0.000001 | Train Accuracy: 0.999444 | Test Accuracy: 0.987048, 0.976624\n",
      "Step 5 | Training Loss: 0.001754 | Train Accuracy: 0.999286 | Test Accuracy: 0.969304, 0.942869\n",
      "Step 1 | Training Loss: 0.000001 | Train Accuracy: 0.999047 | Test Accuracy: 0.969260, 0.942785\n",
      "Step 2 | Training Loss: 0.000001 | Train Accuracy: 0.999762 | Test Accuracy: 0.969482, 0.943207\n",
      "Step 3 | Training Loss: 0.002556 | Train Accuracy: 0.999365 | Test Accuracy: 0.969482, 0.943207\n",
      "Step 4 | Training Loss: 0.000009 | Train Accuracy: 0.999365 | Test Accuracy: 0.969482, 0.943207\n",
      "Step 5 | Training Loss: 0.000005 | Train Accuracy: 0.999603 | Test Accuracy: 0.969482, 0.943207\n",
      "Step 1 | Training Loss: 0.000884 | Train Accuracy: 0.999603 | Test Accuracy: 0.969482, 0.943207\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 0.999444 | Test Accuracy: 0.969438, 0.943122\n",
      "Step 3 | Training Loss: 0.000000 | Train Accuracy: 0.999365 | Test Accuracy: 0.969482, 0.943207\n",
      "Step 4 | Training Loss: 0.000006 | Train Accuracy: 0.999444 | Test Accuracy: 0.969526, 0.943291\n",
      "Step 5 | Training Loss: 0.000003 | Train Accuracy: 0.999603 | Test Accuracy: 0.969615, 0.943460\n",
      "7min 13s ± 56 s per loop (mean ± std. dev. of 10 runs, 1 loop each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit -r 10\n",
    "\n",
    "Hyperparameters.start_training()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-16T20:27:11.593856Z",
     "start_time": "2017-06-16T20:27:11.553335Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-16T20:27:11.600659Z",
     "start_time": "2017-06-16T20:27:11.595499Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-23T21:43:45.588305Z",
     "start_time": "2017-06-23T21:43:45.565921Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>no_of_features</th>\n",
       "      <th>hidden_layers</th>\n",
       "      <th>train_score</th>\n",
       "      <th>test_score</th>\n",
       "      <th>test_score_20</th>\n",
       "      <th>time_taken</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999512</td>\n",
       "      <td>0.999072</td>\n",
       "      <td>117.723715</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   epoch  no_of_features  hidden_layers  train_score  test_score  \\\n",
       "3     18               1              3          1.0    0.999512   \n",
       "\n",
       "   test_score_20  time_taken  \n",
       "3       0.999072  117.723715  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g = df_results.groupby(by=['no_of_features'])\n",
    "idx = g['test_score'].transform(max) == df_results['test_score']\n",
    "df_results[idx].sort_values(by = 'test_score', ascending = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-23T21:43:45.604695Z",
     "start_time": "2017-06-23T21:43:45.590117Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>no_of_features</th>\n",
       "      <th>hidden_layers</th>\n",
       "      <th>train_score</th>\n",
       "      <th>test_score</th>\n",
       "      <th>test_score_20</th>\n",
       "      <th>time_taken</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.999512</td>\n",
       "      <td>0.999072</td>\n",
       "      <td>117.723715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.999468</td>\n",
       "      <td>0.998987</td>\n",
       "      <td>54.473063</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0.999047</td>\n",
       "      <td>0.994455</td>\n",
       "      <td>0.990717</td>\n",
       "      <td>52.805204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999841</td>\n",
       "      <td>0.933907</td>\n",
       "      <td>0.874262</td>\n",
       "      <td>38.998433</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.928007</td>\n",
       "      <td>0.863038</td>\n",
       "      <td>19.724738</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   epoch  no_of_features  hidden_layers  train_score  test_score  \\\n",
       "3     18               1              3     1.000000    0.999512   \n",
       "2      6               1              3     1.000000    0.999468   \n",
       "4      6               1              5     0.999047    0.994455   \n",
       "1     12               1              1     0.999841    0.933907   \n",
       "0      6               1              1     1.000000    0.928007   \n",
       "\n",
       "   test_score_20  time_taken  \n",
       "3       0.999072  117.723715  \n",
       "2       0.998987   54.473063  \n",
       "4       0.990717   52.805204  \n",
       "1       0.874262   38.998433  \n",
       "0       0.863038   19.724738  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_results.sort_values(by = 'test_score', ascending = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-23T21:43:45.639440Z",
     "start_time": "2017-06-23T21:43:45.606696Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.Panel(Train.predictions).to_pickle(\"dataset/tf_GRU_nsl_kdd_predictions.pkl\")\n",
    "df_results.to_pickle(\"dataset/tf_GRU_nsl_kdd_scores.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-23T21:43:45.721218Z",
     "start_time": "2017-06-23T21:43:45.641607Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import itertools\n",
    "\n",
    "def plot_confusion_matrix(cm, classes,\n",
    "                          normalize=False,\n",
    "                          title='Confusion matrix',\n",
    "                          cmap=plt.cm.Blues):\n",
    "    \"\"\"\n",
    "    This function prints and plots the confusion matrix.\n",
    "    Normalization can be applied by setting `normalize=True`.\n",
    "    \"\"\"\n",
    "    np.set_printoptions(precision=4)\n",
    "\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes, rotation=45)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "        print(\"Normalized confusion matrix\")\n",
    "    else:\n",
    "        print('Confusion matrix, without normalization')\n",
    "\n",
    "    print(cm)\n",
    "\n",
    "    thresh = cm.max() / 2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, cm[i, j].round(4),\n",
    "                 horizontalalignment=\"center\",\n",
    "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "\n",
    "def plot(actual_value, pred_value):\n",
    "    from sklearn.metrics import confusion_matrix\n",
    "    cm_2labels = confusion_matrix(y_pred = pred_value, y_true = actual_value)\n",
    "    plt.figure(figsize=[6,6])\n",
    "    plot_confusion_matrix(cm_2labels, preprocess.output_columns_2labels, normalize = False,\n",
    "                         title = Train.best_parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-23T21:43:47.111883Z",
     "start_time": "2017-06-23T21:43:45.723719Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix, without normalization\n",
      "[[ 9710     1]\n",
      " [    0 12833]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbcAAAGeCAYAAAAXNE8VAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmclWX9//HXG4ZVBMSFEFBcSyU3FM3KH6Up5tqiYu6a\nmsvXyiw1M7VvlJUtWqnpVwO1VFxKLAyXMldAxAVxSVwBEUEUUQRh+Pz+uK/Bm2k2Zs7MmbnP++nj\nPOac696ucxjncz6f+7rvSxGBmZlZkXQqdwfMzMxKzcHNzMwKx8HNzMwKx8HNzMwKx8HNzMwKx8HN\nzMwKx8HNzMwKx8HNzMwKx8HNzMwKp6rcHTAzs9Lq3HvjiBUflGx/8cH8iRExsmQ7bAMObmZmBRMr\nPqDbxw8p2f6WPvH79Uq2szbi4GZmVjgCVfZZp8p+92ZmVkjO3MzMikaAVO5elJWDm5lZEbksaWZm\nVizO3MzMishlSTMzKxaPlqzsd29mZoXkzM3MrIhcljQzs0IRLkuWuwNmZmal5szNzKxw5LJkuTtg\nZmatwGVJMzOzYnHmZmZWRC5LmplZsfgi7sp+92ZmVkjO3MzMisZT3ji4mZkVksuSZmZmxeLgZmZW\nOGlASakejR1NukbSm5KezrX9QtJzkp6S9BdJfXPLzpE0U9LzkvbOtQ+TND0tu1TKaquSukm6KbVP\nljSksT45uJmZFVEnle7RuDHAyFptdwNDI2Jb4D/AOQCStgZGAdukbS6T1DltczlwArBFetTs83jg\n7YjYHPg18LNG335Tem1mZlafiLgfWFir7a6IWJFeTgIGpecHAjdGxLKIeBmYCQyXNADoHRGTIiKA\na4GDctuMTc9vAfaoyerq4wElZmZFU/pZAdaTNDX3+sqIuHINtj8OuCk9H0gW7GrMTm3L0/Pa7TXb\nzAKIiBWSFgHrAgvqO6CDm5lZEZX2UoAFEbFT87qhc4EVwJ9K2aHGuCxpZmatQtIxwH7A4anUCDAH\nGJxbbVBqm8NHpct8+2rbSKoC+gBvNXRsBzczs8Jp29GSdfZAGgl8DzggIpbkFo0HRqURkJuQDRyZ\nEhFzgXcl7ZrOpx0F3J7b5uj0/KvAP3PBsk4uS5qZWYtIugEYQXZubjZwPtnoyG7A3Wnsx6SI+EZE\nzJA0DniGrFx5akRUp12dQjbysgdwZ3oAXA1cJ2km2cCVUY32qZHgZ2ZmHUyn3oOi2y7/U7L9Lb3n\n7Meae86tXJy5mZkVkW+/ZWZmVizO3MzMikbyrADl7oCZmbUClyXNzMyKxZmbmVkRuSxpZmbFIpcl\ny90BaxuSZkgaUc+yEenCy/q2HSPpx63WOTOzEnNwKwBJr0jas1bbMZIerHkdEdtExH1t3rkG1O5j\neyfpc2kixXckvZUmYBzY+JYgaYikkPRe7vFkCfp0gaTrW7qfUpG0paSbJS2QtChNVHlGbr6u1jpu\no1/AJJ0maaqkZZLGtGZ/2oWaEZOleHRADm5WsZRZk/8HngG+CKwDbAi8QDa54proGxG90mO7Ndy2\n5NJNaEu1r82AyWRTk3wyIvoABwPDgLVLdZwWeB34MXBNuTvS6mqmvCnjvSXLrWP22tZYPruT1CN9\n031b0jPAzrXW3UHSNEmLJd0EdK+1fD9JT6QM5mFJ29Y6zpnpG/uiNDX8ats3sb/HSno29eElSSfl\nlj0taf/c6y4pU9ghvd419esdSU/my7GS7pM0WtJDwBJg05RBvpSO9bKkw+vqU0TMi4hZuRu2VgOb\nr+l7q+f9Hpfe79uSJkraOLfsEkmzJL0r6TFJn03tI4HvA4fmM8HamXw+u8tlkMdLeg34ZxM+syZ9\nPsCFwMMRcUa6CS4R8XxEHB4R76R9HZBK5O+kf4utcscJSZvnXq/KxpRK55K+I+lNSXMlHZuWnQgc\nDnwvfQ531NW5iLgtIv5KI3eTt2JwcKtM5wObpcfefHS3bSR1Bf4KXAf0A24GvpJbvgPZN9+TyCYL\n/AMwXlK33P4PIZsefhNgW+CYZvTxTbKpMnoDxwK/lrRjWnYtcERu3S8CcyPi8VQm/DvZN/R+wJnA\nrZLWz61/JHAiWTYxH7gU2Cci1gZ2A55I73Wj9Ed4o9z730jSO8AHad8/b8Z7W42kA8mC1JeB9YEH\ngBtyqzwKbJ/ez5+BmyV1j4h/AD8BbmpGJvj/gK2AvRv6zCStRT2fTx32JJslub73uWV6X99K73MC\ncEf6nWuKj5FNdTIQOB74vaR10qSZfwJ+nj6H/dPxLpN0WRP3XTDlnxWg3Dpmr60uf01/iN9Jf3wb\n+p/6EGB0RCyMiFlkf7xq7Ap0AX4TEcsj4hayP641TgT+EBGTI6I6IsYCy9J2NS6NiNcjYiFwB9kf\n5jUSEX+PiBcj82/gLuCzafH1wBcl9U6vjyQLxpAFvQkRMSEiVkbE3cBUsgBYY0xEzIiIFWR3JV8J\nDJXUIyLmRsSM1IfXIqJvRLyW69drEdEXWA/4AfDcGr61Bbl/pzNT2zeAn0bEs6lPPwG2r8neIuL6\niHgrIlZExC/J7rT+8TU8bm0XRMT7EfEBjX9mdX4+dVgXmNvAMQ8F/h4Rd0fEcuBisru/79bEPi8H\nfpR+LycA79HA5xARp0TEKU3cd/H4nJsVxEHpD3Hf9Me3of+pNyRN2Z68WmvZnFpzJeWXbwx8p1Yg\nHZy2q/FG7vkSoNeavBEASftImiRpYTrGF8kCChHxOvAQ8BVJfYF9+GiW342Bg2v17zPAgNzuV733\niHif7I/uN4C5kv4u6RON9S8F7rHA7Vqz81br5f6dLs71+ZJcfxeSnTUZmD6LM1PJclFa3qfms2iB\n/L9/vZ/ZGn4+b7H651zbhuR+lyJiZepHkwblAG+l4F+jWb9bVhkc3CrTXFafCXejWssGSqt9Xcsv\nn0WW9fXNPXpGRL6M1iKpxHkr2Tf7/ilYTyD7g19jLFnGcTDwSETUzNg7C7iuVv/WioiLctuuNs9T\nREyMiC+Q/WF+DriqiV2tAjYgK522xCzgpFp97hERD6fza98jy7bXSZ/FIj76LOqas+p9oGfu9cfq\nWCe/XYOf2Rp8PveQK2HX4XWyQApkA3rIfg9r/u2WNKHf9fHcXbW5LGkVaBxwjqR1JA0C8hM/PUJW\nqjtd2UCNLwPDc8uvAr4haRdl1pK0r6TmjoaTpO75B9CVrPQ2H1ghaR9gr1rb/RXYEfgm2Tm4GtcD\n+0vaW1LntM8R6X3WdfD+kg5M55aWkZW6Vtaz7pclfVxSp3QO71fA4ymLqxm4cV8zPoMryP49tkn7\n6SPp4LRsbbJ/j/lAlaQfsnownQcM0eqjPp8gm+m4i6SdyGYubki9n9mafD5k53J3k/QLSR9L72Vz\nSdenDHscsK+kPSR1Ab6T9vlwrt9fS30YSXZesKnmAZs2tIKkqvT71RmoeZ/FvZGFy5JWgS4kKw+9\nTHYuq+Z8FRHxIdnAhmPIymOHArfllk8FTgB+B7wNzKR5A0Zq7EY2OKP243SyP4ZvA18jm2Z+lXSu\n6FayQSv5/s0CagZozCfLSr5L/b/rnYAzyLKKhWR/UE+GVYNH3ssNKBkI/ANYDEwn+yP/pdy+BpOV\nS9dIRPwF+Blwo6R3gafJSq0AE9Mx/0P2b7aU1UuKN6efb0malp6fRzZY6G2yf+s/N3L8hj6zej+f\nOvbzIvApYAgwQ9Iisn+jqcDiiHieLNv+LbAA2B/YP/3OQfZFZX/gHbLRj39tqN+1XA1sncqqfwWQ\ndIWkK3Lr/IDsd+vs1I8PUpsVkGfitg4rZTFbRsQRja7cBiQ9AewRER5qbmXVaZ0h0W1E6eL20r+e\n4Jm4zdqCpH5kw8GPLHdfakTEGo8KNWs1HbScWCouS1qHI+kEstLZnRFxf7n7Y2btjzM363Ai4iqa\nPqLRrCKpwjM3Bzczs4IRDm4uS5qZWeE4c2umzj36RFXvDcrdDesghg7qU+4uWAfx6quvsGDBgpal\nXWL1Wx5UIAe3ZqrqvQEDDvt1ubthHcRDF+9X7i5YB/HpXUox4l4uS5a7A2ZmZqXmzM3MrIAqPXNz\ncDMzK6BKD24uS5qZWeE4czMzK6BKz9wc3MzMisaXArgsaWZmxePMzcysYOTr3BzczMyKqNKDm8uS\nZmZWOM7czMwKqNIzNwc3M7MCqvTg5rKkmZkVjjM3M7Oi8XVuDm5mZkXksqSZmVnBOHMzMysYX8Tt\n4GZmVkiVHtxcljQzs8Jx5mZmVkSVnbg5uJmZFY5clnRZ0szMCseZm5lZAVV65ubgZmZWQJUe3FyW\nNDOzwnHmZmZWML6I25mbmVkxqYSPxg4lXSPpTUlP59r6Sbpb0gvp5zq5ZedIminpeUl759qHSZqe\nll2qFKEldZN0U2qfLGlIY31ycDMzs5YaA4ys1XY2cG9EbAHcm14jaWtgFLBN2uYySZ3TNpcDJwBb\npEfNPo8H3o6IzYFfAz9rrEMObmZmRZOucyvVozERcT+wsFbzgcDY9HwscFCu/caIWBYRLwMzgeGS\nBgC9I2JSRARwba1tavZ1C7CHGumYz7mZmRVQOzjn1j8i5qbnbwD90/OBwKTcerNT2/L0vHZ7zTaz\nACJihaRFwLrAgvoO7uBmZmaNWU/S1NzrKyPiyqZuHBEhKVqhX/VycDMzK6ASZ24LImKnNdxmnqQB\nETE3lRzfTO1zgMG59Qaltjnpee32/DazJVUBfYC3Gjq4z7mZmRVRG46WrMd44Oj0/Gjg9lz7qDQC\nchOygSNTUgnzXUm7pvNpR9XapmZfXwX+mc7L1cuZm5mZtYikG4ARZOXL2cD5wEXAOEnHA68ChwBE\nxAxJ44BngBXAqRFRnXZ1CtnIyx7AnekBcDVwnaSZZANXRjXWJwc3M7MCassBJRFxWD2L9qhn/dHA\n6DrapwJD62hfChy8Jn1ycDMzK5imDuEvMp9zMzOzwnHmZmZWQJWeuTm4mZkVUKUHN5clzcyscJy5\nmZkVUWUnbg5uZmZF5LKkmZlZwThzMzMrGjlzc3AzMysYARUe21yWNDOz4nHmZmZWOL79loObmVkB\nVXhsc1nSzMyKx5mbmVkBuSxpZmbFIpclXZY0M7PCceZmZlYwAjp1quzUzZmbmZkVjjM3M7MCqvRz\nbg5uZmYFVOmjJV2WNDOzwnHmZmZWNL4UwMHNzKxoslkBKju6uSxpZmaF48zNGnTs7psw6lODEeLG\nSa9xzb9f5ndH78imG6wFQO8eXXj3g+V88RcP0LdnFy4/dhjbbtSXW6bM5vxbn161n6GD+nDx17aj\ne5fO/OvZN7nwthnlektWZid9/TjunPA31t9gAx574unGN7Bm8KwADm5Wry0/tjajPjWYA3/1IMur\ng7EnDefeGfM4bey0Veuce+BWLF66AoBlK1byywnP8/EBa7PlgN6r7evHB3+Sc256isdffYcxJw1n\nxFbrc9+z89v0/Vj7cOTRx/CNU07j68cdVe6uFFqFxzaXJa1+m/fvxROvvsPS5SupXhlMfnEhI7cd\nsNo6+26/IeMfex2ADz6sZurLb7NsxcrV1lm/dzfW7l7F46++A8Btj85mr09+rG3ehLU7n/ns7vTr\n16/c3bCCc3Czej3/xmJ23rQffXt2oXuXTnxu6w0Y0Lf7quXDN+3HgsXLeGXB+w3u52N9ujP3nQ9W\nvZ77zlL69+newBZm1lKSSvboiFyWtHq9OO89rrj3Ra47eReWfFjNM3MWsTJi1fIDhm3I+Gmvl7GH\nZlYnXwrQepmbpIebsc0rkm7Nvf6qpDEl7VjjfbhA0pltecz2bNzkWez/ywc59LePsGjJcl56M8vS\nOncSe287gL893nhwe2PRUgb07bHq9YC+3Zm3aGmr9dnMrNWCW0Ts1sxNh0naujkbSnImWmLr9uoK\nwIZ9uzNy2wGMnzYHgM9suR4vzXuPN5oQpOa/u4zFS1eww8Z9AfjyzoO4a/q81uu0WYWruc7NZclW\nIOm9iOglaQBwE9A7He/kiHiggU1/CZwLHF5rf/2Aa4BNgSXAiRHxlKQLgM1S+2uSJgIHAWsBWwAX\nA12BI4FlwBcjYqGkE4AT07KZwJERsaQkb75ALj92GOus1ZUV1cF5t0zn3Q+ykZH777jhqkCX9+AP\nP0+vblV0qerEXp/sz5GXT2bmvPc475bpqy4FuO/Z+dz37Jtt/VasnTjqiMN44N/3sWDBAjYbMojz\nfnghxxx3fLm7VTgdNCaVTFtkOl8DJkbEaEmdgZ6NrD8OOEXS5rXaLwQej4iDJH0euBbYPi3bGvhM\nRHwg6RhgKLAD0J0scJ0VETtI+jVwFPAb4LaIuApA0o+B44HfNtQxSSeSBUQ6r71+4++8AA757SN1\ntp/55yfrbP/Mj/5ZZ/v0WYvY+2f3l6xf1nFde/0N5e6CVYC2CG6PAtdI6gL8NSKeaGT9auAXwDnA\nnbn2zwBfAYiIf0paV1LNxVTjI+KD3Lr/iojFwGJJi4A7Uvt0YNv0fGgKan2BXsDExt5IRFwJXAnQ\nrf8W0cjqZmZl01HLiaXS6pcCRMT9wO7AHGCMpKZcuXld2mZwEw9Teyz6stzzlbnXK/kooI8BTouI\nT5JlhR6bbmaFIZXu0RG1enCTtDEwL5UA/w/YsbFtImI58Gvg27nmB0jn4SSNABZExLst6NrawNyU\nUR7e2MpmZtZxtEVZcgTwXUnLgffIznk1xdXAD3KvLyArbz5FNqDk6Bb26zxgMjA//Vy7hfszM2sf\n5LJkqwW3iOiVfo4FxjZxmyG558uADXOvF5KNgqy9zQW1Xo8hKznWtc9VyyLicuDyxvZnZtbRZJcC\nlLsX5eXbb5mZWeGU5aJnSZOBbrWaj4yI6eXoj5lZsXTci69LpSzBLSJ2KcdxzcwqRYXHNpclzcys\neHwvRjOzAnJZ0szMiqUDX3xdKi5LmplZ4ThzMzMrmJopbyqZg5uZWQFVenBzWdLMzArHmZuZWQFV\neOLm4GZmVkQuS5qZmbWApG9LmiHpaUk3SOouqZ+kuyW9kH6uk1v/HEkzJT0vae9c+zBJ09OyS9WC\nCO3gZmZWNCWcqLSx8CJpIHA6sFNEDAU6A6OAs4F7I2IL4N70Gklbp+XbACOByyR1Tru7HDgB2CI9\nRjb3I3BwMzMrGKUbJ5fq0QRVQA9JVUBP4HXgQD6a7mwsH01ZdiBwY0Qsi4iXgZnAcEkDgN4RMSki\nAriWOqY5ayoHNzMza7aImANcDLwGzAUWRcRdQP+ImJtWewPon54PBGbldjE7tQ1Mz2u3N4uDm5lZ\nAZW4LLmepKm5x4kfHUfrkGVjm5BNML2WpCPyfUmZWLTdu/doSTOzQupU2tGSCyJip3qW7Qm8HBHz\nASTdBuwGzJM0ICLmppLjm2n9OcDg3PaDUtuc9Lx2e7M4czMzs5Z4DdhVUs80unEP4FlgPHB0Wudo\n4Pb0fDwwSlI3SZuQDRyZkkqY70raNe3nqNw2a8yZm5lZAbXVZW4RMVnSLcA0YAXwOHAl0AsYJ+l4\n4FXgkLT+DEnjgGfS+qdGRHXa3SnAGKAHcGd6NIuDm5lZwWTnytruIu6IOB84v1bzMrIsrq71RwOj\n62ifCgwtRZ9cljQzs8Jx5mZmVkCdKvvuWw5uZmZF5HtLmpmZFYwzNzOzAqrwxM3BzcysaER2f8lK\n5rKkmZkVjjM3M7MC8mhJMzMrlqZPVVNYLkuamVnhOHMzMyugCk/cHNzMzIpGlHzKmw7HZUkzMysc\nZ25mZgVU4Ymbg5uZWRF5tKSZmVnBOHMzMyuYbLLScveivBzczMwKyKMlzczMCqbezE1S74Y2jIh3\nS98dMzMrhcrO2xouS84AgtU/o5rXAWzUiv0yM7MWqPTRkvUGt4gY3JYdMTMzK5UmnXOTNErS99Pz\nQZKGtW63zMysubLbb5Xu0RE1Gtwk/Q74HHBkaloCXNGanTIzsxZIU96U6tERNeVSgN0iYkdJjwNE\nxEJJXVu5X2ZmZs3WlOC2XFInskEkSFoXWNmqvTIzsxbpoAlXyTQluP0euBVYX9KFwCHAha3aKzMz\na5GOWk4slUaDW0RcK+kxYM/UdHBEPN263TIzM2u+pt5+qzOwnKw06buamJm1YzWjJStZU0ZLngvc\nAGwIDAL+LOmc1u6YmZk1n0dLNu4oYIeIWAIgaTTwOPDT1uyYmZlZczUluM2ttV5VajMzs3aqY+Zb\npdPQjZN/TXaObSEwQ9LE9Hov4NG26Z6Zma0pyVPeNJS51YyInAH8Pdc+qfW6Y2Zm1nIN3Tj56rbs\niJmZlU6FJ26Nn3OTtBkwGtga6F7THhFbtmK/zMzMmq0p16yNAf5Idn5yH2AccFMr9snMzFqo0i8F\naEpw6xkREwEi4sWI+AFZkDMzs3ZKKt2jI2rKpQDL0o2TX5T0DWAOsHbrdsvMzKz5mhLcvg2sBZxO\ndu6tD3Bca3bKzMyaT8iXAjS2QkRMTk8X89GEpWZm1l514HJiqTR0EfdfSHO41SUivtwqPTIzM2uh\nhjK337VZLzqgoYP68NDF+5W7G9ZBrLPzaeXugnUQy55/rST76aijHEuloYu4723LjpiZWelU+txk\nlf7+zcysgJo6WamZmXUQwmXJJgc3Sd0iYllrdsbMzErDM3E3QtJwSdOBF9Lr7ST9ttV7ZmZm1kxN\nOed2KbAf8BZARDwJfK41O2VmZi3TSaV7dERNKUt2iohXa9Vvq1upP2Zm1kLZPSE7aFQqkaYEt1mS\nhgMhqTPwP8B/WrdbZmZmzdeUsuTJwBnARsA8YNfUZmZm7VRbliUl9ZV0i6TnJD0r6VOS+km6W9IL\n6ec6ufXPkTRT0vOS9s61D5M0PS27VC1IPxsNbhHxZkSMioj10mNURCxo7gHNzKz1tfGUN5cA/4iI\nTwDbAc8CZwP3RsQWwL3pNZK2BkYB2wAjgctSVRDgcuAEYIv0GNnc99+Umbivoo57TEbEic09qJmZ\nFYOkPsDuwDEAEfEh8KGkA4ERabWxwH3AWcCBwI3p0rKXJc0Ehkt6BegdEZPSfq8FDgLubE6/mnLO\n7Z7c8+7Al4BZzTmYmZm1PkFbTnmzCTAf+KOk7YDHgG8C/SNiblrnDaB/ej4QmJTbfnZqW56e125v\nlqZMeXNT/rWk64AHm3tAMzNrfSW+t+J6kqbmXl8ZEVem51XAjsD/RMRkSZeQSpA1IiIk1TvLTGto\nzu23NuGjCGxmZsW3ICJ2qmfZbGB2bu7PW8iC2zxJAyJirqQBwJtp+RxgcG77QaltTnpeu71ZmnKH\nkrclLUyPd4C7gXOae0AzM2t9bTWgJCLeILtk7OOpaQ/gGWA8cHRqOxq4PT0fD4yS1E3SJmQDR6ak\nEua7knZNoySPym2zxhrM3NIBtuOj6LkyIto0tTQzszUjqS3PuUF2/fOfJHUFXgKOJUuexkk6HngV\nOAQgImZIGkcWAFcAp0ZEzY1BTgHGAD3IBpI0azAJNBLcUp10QkQMbe4BzMys2CLiCaCusuUe9aw/\nGhhdR/tUoCTxpinnHJ+QtEMpDmZmZm2jja9za3fqzdwkVUXECmAH4FFJLwLvk40yjYjYsY36aGZm\na6ij3vC4VBoqS04hG955QBv1xczMrCQaCm4CiIgX26gvZmZWAm18EXe71FBwW1/SGfUtjIhftUJ/\nzMysBCo8tjUY3DoDvUgZnJmZWUfRUHCbGxE/arOemJlZaXTgGbRLpdFzbmZm1vGowv+EN3SdW50X\n35mZmbV39WZuEbGwLTtiZmalkY2WLHcvyqs5swKYmVk7V+nBrcRT/piZmZWfMzczswJShV/o5uBm\nZlYwPufmsqSZmRWQMzczs6LpwFPVlIqDm5lZAVX6jZNdljQzs8Jx5mZmVjAeUOLgZmZWSBVelXRZ\n0szMiseZm5lZ4YhOFT4rgIObmVnBCJclXZY0M7PCceZmZlY0nonbwc3MrIh8EbeZmVnBOHMzMysY\nDyhxcDMzKySXJc3MzArGmZuZWQFVeOLm4GZmVjTCZblKf/9mZlZAztzMzIpGoAqvSzq4mZkVUGWH\nNpclzcysgJy5mZkVTDYTd2Xnbg5uZmYFVNmhzWVJMzMrIGduZmYFVOFVSQc3M7PiUcVfCuCypJmZ\nFY4zNzOzgvHttxzczMwKyWVJMzOzgnHmZiVx18R/cOYZ36S6uppjjvs63/3e2eXukrWBK84/nH12\nH8r8hYvZ6eCfAPCTbx3EF3cfyofLq3l59gJOPP96Fr33AVVVnbj8h4ez/ScGU9W5E3/6+xQuvuYu\nAG7/3Sl8bP3eVHXuzEOPv8i3fnoTK1cGX//qZzjpkN2pXrmS95cs49Qf38BzL71RzrfcYVR23ubM\nzUqgurqab51+KrffcSePP/UMN994A88+80y5u2Vt4Lo7JnHgqb9fre3eSc8x7OCfMPzQn/LCq2/y\n3eP2AuAre+5It65V7HzIT9jt8J/x9a98mo0G9APgiLOuYZdDL2LYV0ez/jq9+MoXdgTgpjunsvMh\nP2HXURfxq7H38LMzvty2b7CjSjdOLtWjI3JwsxZ7dMoUNttsczbZdFO6du3KwYeO4m933F7ublkb\neGjaiyxctGS1tnsnPUd19UoApkx/mYH9+wIQBD27d6Vz50706NaVD5dXs/j9pQCrflZVdaJLVWci\nYrV2gLV6dCWIVn9PVgwuS1qLvf76HAYNGrzq9cCBg5gyZXIZe2TtxVEHfopb7poGwG33PM5+I7bl\n5btH07N7V7538W28/e5HgXH8709lp6Ebc9dDz3DbPY+vaj/pkN05/YjP0bVLFSNPurTN30NH5NGS\nfv9m1kq+d/zeVFev5MYJjwKw8zZDqK5eyaZ7nctW+57PN4/8PEMGrrtq/QNO/T2bfOH7dOtaxYid\nP76q/Q/j7mebAy7kB5fcztlfH9nm76OjclmyjUh6uJnbbS8pJI3MtfWVdEru9RBJX2tB3+6TtFNz\nt690G244kNmzZ616PWfObAYOHFjGHlm5HbH/Lnxx96Ecc+6YVW2H7LMTdz38DCtWrGT+2+/xyBMv\nMWzrjVbbbtmHK7jjvqfYf8Qn/2uf4yY+xv4jtm3trltBtFlwi4jdmrnpYcCD6WeNvsApuddDgGYH\nN2uZnXb+6NcbAAAU1UlEQVTemZkzX+CVl1/mww8/5OabbmTf/Q4od7esTL6w21acccyefPVbf+CD\npctXtc9+Y+GqjKxn964M33YIz78yj7V6dOVj6/UGoHPnTuzzmW14/pV5AGy20fqrtt/ns9swc9b8\nNnwnHZtK+OiI2uycm6T3IqKXpAHATUDvdPyTI+KBerYRcDDwBeABSd0jYilwEbCZpCeAu4HPAlul\n12OBvwDXAWulXZ0WEQ+nfZ4FHAGsBO6MiLNzx+sEXAPMjogf1NGfE4ETAQZvtFHtxRWrqqqKX1/y\nO/bfd2+qq6s5+pjj2HqbbcrdLWsDY396DJ8dtgXr9e3FzH/8L/97xQS+e+xedOtaxd8uPw2AKdNf\n4fTRN3LFTfdz5YVH8Ngt5yLBdbdP4ukXXmeDfmtzy29OomuXKjp1EvdPfYGrbnkQgJMP3Z3P7fIJ\nlq+o5p13l3DCedeW8+12KG1dTZTUGZgKzImI/ST1I/tbPwR4BTgkIt5O654DHA9UA6dHxMTUPgwY\nA/QAJgDfjJrRRWvan2Zut+YH+ii4fQfoHhGj04fRMyIW17PNp4EfRcQekv4M3BoRt0oaAvwtIoam\n9UYAZ0bEful1T2BlRCyVtAVwQ0TsJGkf4Dxgz4hYIqlfRCyUdB9wNvBN4OmIGN3Y+xk2bKd4aPLU\nFn0mVjnW2fm0cnfBOohlz49j5ZI3WxSaNt9mu/jljRNL1SUO2nbAYxHR4KkbSWcAOwG9U3D7ObAw\nIi6SdDawTkScJWlr4AZgOLAhcA+wZURUS5oCnA5MJgtul0bEnc3pczkGlDwKHCvpAuCT9QW25DDg\nxvT8RlYvTTakC3CVpOnAzcDWqX1P4I8RsQQgIhbmtvkDTQxsZmbtWTZaUiV7NHo8aRCwL/B/ueYD\nySpppJ8H5dpvjIhlEfEyMBMYnqp6vSNiUsrWrs1ts8baPLhFxP3A7sAcYIyko+paL2V1XwF+KOkV\n4LfASElrN+Ew3wbmAduRfZPo2oRtHgY+J6l7E9Y1M6sk60mamnucWGv5b4DvkZ3uqdE/Iuam528A\n/dPzgcCs3HqzU9vA9Lx2e7O0eXCTtDEwLyKuIovyO9az6h7AUxExOCKGRMTGwK3Al4DFQD7I1X7d\nB5gbESuBI4HOqf1usqyxZ+pLv9w2V5OlweMk+fo/M+vQpNI9gAURsVPuceVHx9F+wJsR8Vh9fUmZ\nWJtegV+OsuQI4ElJjwOHApfUs95hZAND8m4FDouIt4CHJD0t6RfAU0C1pCclfRu4DDha0pPAJ4D3\nASLiH8B4YGoafHJmfucR8SvgceC6NLjEzKwDUkn/a8SngQNShe1G4POSrgfmpVIj6eebaf05wODc\n9oNS25z0vHZ78z6BthpQUjQeUGJrwgNKrKlKMaBki222j9/cdFepusR+n+zf6IASWH1wX0o83soN\nKOkXEd+TtA3wZz4aUHIvsEU9A0p+GxETmtNnl9/MzAqoHdxY5CKy0zzHA68ChwBExAxJ44BngBXA\nqRFRnbY5hY8uBbgzPZqlXQQ3SZOBbrWaj4yI6eXoj5lZR1YzWrKtRcR9wH3p+VtkYyfqWm808F8j\n0yNiKjC0FH1pF8EtInYpdx/MzKw42kVwMzOzElK7KEuWlYObmVkBVXpw83B3MzMrHGduZmYF1ITr\n0wrNwc3MrGAEdKrs2OaypJmZFY8zNzOzAnJZ0szMCsejJc3MzArGmZuZWQG5LGlmZoXi0ZIuS5qZ\nWQE5czMzK5wmTTJaaA5uZmZF4xsnuyxpZmbF48zNzKyAKjxxc3AzMyuabLRkZYc3lyXNzKxwnLmZ\nmRVQZedtDm5mZsVU4dHNZUkzMyscZ25mZgXki7jNzKxwKnywpMuSZmZWPM7czMwKqMITNwc3M7NC\nqvDo5rKkmZkVjjM3M7OCER4t6eBmZlY0nvLGZUkzMyseZ25mZgVU4Ymbg5uZWSFVeHRzWdLMzArH\nmZuZWeHIoyXL3QEzMys9j5Y0MzMrGGduZmYFIyp+PImDm5lZIVV4dHNZ0szMCseZm5lZAXm0pJmZ\nFY5HS5qZmRWMMzczswKq8MTNwc3MrHB8LYDLkmZmVjzO3MzMCsijJc3MrFCER0u6LGlmZoXjzM3M\nrIAqPHFzcDMzK6QKj24uS5qZWbNJGizpX5KekTRD0jdTez9Jd0t6If1cJ7fNOZJmSnpe0t659mGS\npqdll0rNP3Po4GZmVkAq4X+NWAF8JyK2BnYFTpW0NXA2cG9EbAHcm16Tlo0CtgFGApdJ6pz2dTlw\nArBFeoxs7vt3cDMzKyCpdI+GRMTciJiWni8GngUGAgcCY9NqY4GD0vMDgRsjYllEvAzMBIZLGgD0\njohJERHAtblt1piDm5mZlYSkIcAOwGSgf0TMTYveAPqn5wOBWbnNZqe2gel57fZm8YASM7MCKvF4\nkvUkTc29vjIirlzteFIv4FbgWxHxbv50WUSEpChtlxrm4GZmVkSljW4LImKneg8ldSELbH+KiNtS\n8zxJAyJibio5vpna5wCDc5sPSm1z0vPa7c3isqSZmTVbGtF4NfBsRPwqt2g8cHR6fjRwe659lKRu\nkjYhGzgyJZUw35W0a9rnUblt1pgzNzOzgskmBWizC90+DRwJTJf0RGr7PnARME7S8cCrwCEAETFD\n0jjgGbKRlqdGRHXa7hRgDNADuDM9msXBzcysaJowyrFUIuJB6i+C7lHPNqOB0XW0TwWGlqJfLkua\nmVnhOHNrpmnTHlvQo4teLXc/2pn1gAXl7oR1GP59qdvGpdhJhd99y8GtuSJi/XL3ob2RNLWhEVVm\nef59aWUVHt1cljQzs8Jx5mZmVjhNuidkoTm4WSld2fgqZqv496UVeSZusxKpfTses4b498VakzM3\nM7OCERU/nsTBzcyskCo8urksaWZmhePMzcpKUj9gvYj4T7n7Yh2HJKUJLa0elT5a0pmblY2k7sDp\nwHGStip3f6z9kzQYsvnByt2X9q6tZuJurxzcrGwiYilwT3p5sKSty9kfa38k9ZLUNT3fCvi5pLXL\n3C3rABzcrCzSfE01dxQfD/QGvuoAZzUkrQX8CTg4NS1Jj/fS5Jirfo/sv6mEj47Iwc3aXM35Ekmb\nSKqKiIeBPwJ9yAKcS5RGRLwP3AQcK+lQYAjwQWSWp3VcnrQ6eUCJtbkU2PYFzgMekPQe8BuyO1Yc\nDxwh6U8R8Uw5+2nlI6lzRFRHxJ8lzQfOAh4DNpF0CTAbWAZU1Zr92aBN53Nrr5y5WZuTtCvwE+BQ\nsi9YBwE/B+YDY4G1gA/L1kErq5TZV0v6gqSfR8TdwCVkE19+CLyWfvYCJpexq+1cZRcmnblZm5HU\nCQiyebyOAj4B7A6cDZwIXEz2Df3cVJKyCpQy+z2Ay4CTUtsdklYAZwD/iYg7ytlHa/+cuVmry530\n75XOl/wtIp4ky9i+HhETgTfJvmz1d2CrXMpUASOB8yLinzWjJSPiTuAK4CxJA8vZz/ZO+FIABzdr\ndblzbPdKukDSl9OiDYATJe0CDAcujoiny9ZRK7v05WcFsBTYVVL3iPgQQNLOwATggIiYU85+dgSV\nXZR0cLM2IGkAcDhZ2XEhsHcKdscBg4EfAj+NiKfK10srl5rMXtJGkgal5juBLsD/S8u2A34NbBkR\nC8vSUetQfM7NWpWknYDtgDkRcZOk9YG9gS8BXSJiP0k9I2KJb6lUmXKZ/U+BhyX1i4hD0iUhR0o6\ni+wykR+ncrY1QUctJ5aKg5u1GkkjyEY/TiQb3n9DREyTdCfQFThQ0pSIeB18zVKlyV3vuCvZaNn9\nyDK1ayTdExF7ShpD9uVoUUS86C9ATVfp95Z0cLNWIWkT4PvAkRFxv6SZwPWSDo+IxyXdDvyjJrBZ\n5Uj3FF2ehvv3B94CDgG2IBsd2Qe4T9LDEbEbMK1mWwc2ayqfc7OSyZ072ZnsG3gfshGRRMTPgauB\n8ZKGRcRbDmyVJ10OshvwLUn7kZ1vXQw8A+wLXBMRi8ky/o3S75I1R4WPKHFws5JJJabdyUpM08ku\n1O4p6bS0/JfA78kuvrXK9RSwF3AdcEtEvEH2J3QusJmkE8hKlF+IiEfL182OrcJjm4OblY6kjwMn\nA2Mi4jHgPuBe4BOSvgMQERdFxL99w9vKImktSYMiYiWwcWr+F7BPGu6/kmyGiCVkge2KiHi2TN21\nAvA5NyulTwL9gT0lTYiI+ZL+QTake4SkjSPiVfC5kwo0BPixpKnAUOA7wNtk9xf9FXAK8BJZwPtJ\nRKzw4JHm68gXX5eKMzdrttw5tkGS+kTELWR/rN4lu7v/uun8yR3AD2sCm1WeiJgBzCQbZDQ5Xaw/\nn+wWW90k3UuW6S9PF3H7C1ALqYT/dUQObtYskjqlc2z7kF1we7Wk+4Fngb8BNdcorRsRi9N5Fasg\nkvpK6plrehr4JXCUpD0i4sN04f65wBjg2xExqQxdtQJyWdLWiKQeEfFBRKyUtDnwv8BJEfGwpEuB\nv5JdpN0l/VyLbKi3VRBJ/YD/APdIeiAifh8RY9OyWcCvJB0NvAN8uWbaGpciS6hjJlwl4+BmTSap\nD3CRpL9ExF1kf5ieI/sjRkScLukG4OyIOF/SoxExt4xdtvJ5G7iLbATk4ZKGAw8CN0fEVZI+BG4F\nVgDfqtnIga10Kjy2uSxpa6Q32XmTr6UpSd4F1gX2zK0zgTQXmwNb5UpBahrZAKPdycqOuwP/lvQ5\nsoEjuwBfSXf7NyspZ27WKElrp/NmsyRdC4wiu+nxfLIBAmMkfQJYlNq/V77eWnsRERdLmkD25edp\nYHuyTH8UsDlwqGeBaD2VPlrSwc0aJGkIcIukx4BxwAvAH4FlZMO5fwYcDOwDbEg2KOAenzupbJI6\nR0Q1Wcb2JbI7+l+dAt4GZDfNXlDOPhZbxx3lWCoObtaY7sAA4EDgFbI7jFwBrAM8TDb0f3REXJLf\nyIGtsqXABjAZuAB4JCIuTm3z/fthrc3n3Kxeabj/c2RlpUXAa8ChwOtk9478anr98zTs279PtkrK\n3l8FzgB61cye7cDW+jwTtzM3a0Aa7t8pIp6VdARwI9ndI66WdAvZXdwPBJ6IiHfK2lkri9y0NZ3S\nLbRWyQWx2cDK/97arPU4uFmDcgHuUUmjgBvSvQB/DzxPdpNkX59UgXKBbQ+yzGxiRCytvV5EPC3p\nrIiYU4ZuWoVyGckalQ9wZGXI8ySdWmsdB7YKkgaMhKSRwOXA23UFNmU6RcSrknpKWrfte1uZKr0s\n6eBmq+TuFflfvxe5APcYsD8wo637Z+UnafN0aUi1pHXIBhR9I01I+1lJR6cLtmt0Sr87fcmubetX\nlo5XoEq/t6TLkgY0rcRUK4NzKbIy9Qc2kDQpIt6W9C/g+DQHWydgOdm52CmSqtLd/fsANwPfjYgX\nytd1qyTO3KzJJaaa1dM2PcguB7AKEhEPkU1E+5Kk3mTXsU0BfhsRh5JdC7mNpK4psK0D/AX4UUTc\nX65+V5wSliRdlrQOZ01LTDUX5qYS031kt96yCpOmMfom2XWOCyLiknTj7M+S3Uj7/yLiw7T6YcCP\nI+KBMnW3IpVyFu4OGttclqxwLjFZs0TE7ZKWA49JGgYsJbvu8QcR8feaknVEXFbenlqlcnCrYBHx\nkKS1yUpM25KVmPYFHk3fxA8Ajk0lpg9TdncrcL6/iVtETJC0kmwOv48DZ0XE0tz5W5+TLaeOmnKV\niMuSFc4lJmuJiPgH8HVgh5rztDUBzYGtvDxa0iqeS0zWEhHxd/DoWWtfHNwMcInJWs6/H+1LRx3l\nWCouS9oqLjGZFYdHS5rluMRkZkXgzM3q5MBm1sG1YeomaaSk5yXNlHR2qd9KczhzMzMroLYa5Sip\nM9kkxl8gm97oUUnjI+KZNulAPZy5mZlZSwwHZkbES+myoRvJ5nksK2duZmYFUzMTdxsZCMzKvZ4N\n7NJmR6+Hg5sVjqRqspv7VpFd2nB0RCxp5r5GAGdGxH7pji1bR8RF9azbF/jaml4PKOkC4L2IuLgp\n7bXWGQP8LSJuaeKxhqT1h65JH61jmTbtsYk9umi9Eu6yu6SpuddXRsSVJdx/yTm4WRF9EBHbA0j6\nE/AN4Fc1C9O8dYqIlWuy04gYD4xvYJW+wCmAL3a3soqIkW14uDnA4NzrQamtrHzOzYruAWBzSUPS\naK5rgaeBwZL2kvSIpGmSbpbUC1aN/HpO0jTgyzU7knSMpN+l5/0l/UXSk+mxG3ARsJmkJyT9Iq33\nXUmPSnpK0oW5fZ0r6T+SHiS7aL5Bkk5I+3lS0q2SeuYW7ylpatrffmn9zpJ+kTv2SS39IM3q8Siw\nhaRNJHUFRtHwl8A24eBmhSWpCtiHrEQJ2QwHl0XENsD7wA+APSNiR2AqcIak7sBVZLONDwM+Vs/u\nLwX+HRHbATuSzUx+NvBiRGwfEd+VtFc65nBge2CYpN3TLc5GpbYvAjs34e3cFhE7p+M9CxyfWzYk\nHWNf4Ir0Ho4HFkXEzmn/J0japAnHMVsjEbECOA2YSPa7OS4iZpS3Vy5LWjH1kPREev4AcDWwIfBq\nRExK7bsCWwMPZVVKugKPAJ8AXq6ZzkfS9cCJdRzj88BRABFRDSxKsybk7ZUej6fXvciC3drAX2rO\nA0pqyrfcoZJ+TFb67EX2h6TGuFRifUHSS+k97AVsK+mraZ0+6dj/acKxzNZIREwAJpS7H3kOblZE\nq8651UgB7P18E3B3RBxWa73VtmshAT+NiD/UOsa3mrGvMcBBEfGkpGOAEblltS+4j3Ts/4mIfBCs\nGVBiVnguS1qlmgR8WtLmAJLWkrQl8BwwRNJmab3D6tn+XuDktG3nNInrYrKsrMZE4LjcubyBkjYA\n7gcOktQjzae3fxP6uzYwV1IX4PBayw6W1Cn1eVPg+XTsk9P6SNpS0lpNOI5ZIThzs4oUEfNTBnSD\npG6p+QcR8R9JJwJ/l7SErKy5dh27+CZwpaTjgWrg5Ih4RNJDkp4G7kzn3bYCHkmZ43vAERExTdJN\nwJPAm2Qn5BtzHjAZmJ9+5vv0GjAF6A18I83m8H9k5+KmpdGh84GDmvbpmHV88i0EzcysaFyWNDOz\nwnFwMzOzwnFwMzOzwnFwMzOzwnFwMzOzwnFwMzOzwnFwMzOzwnFwMzOzwvn/r8XiuDYQ8ZwAAAAA\nSUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fcb913f10b8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot(actual_value = Train.actual_value, pred_value = Train.pred_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-23T21:43:47.442789Z",
     "start_time": "2017-06-23T21:43:47.113663Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix, without normalization\n",
      "[[2151    1]\n",
      " [   0 9698]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbUAAAGgCAYAAAAtsfn1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xm8XfP1//HXO4lIJJHBEGQQYwlqSJCfqYoSNVYrYp5q\n1hpKadHq9yutr6qWtijVJmgRFFFzfetrJokpCY05JILEEEMMyc36/bE/V4/b3Jtzk3PvOWfv99Pj\nPO45e1zn5LrrrLU/e29FBGZmZnnQodoBmJmZVYqTmpmZ5YaTmpmZ5YaTmpmZ5YaTmpmZ5YaTmpmZ\n5YaTmpmZ5YaTmpmZ5YaTmpmZ5UanagdgZmZtq+Oyq0bM/6Ri24tPZt0VEcMrtsEKclIzM8u5mP8J\nS39lRMW29+lTv1++YhurMCc1M7PcE6gYR5uK8S7NzKwQXKmZmeWdAKnaUbQLJzUzsyJw+9HMzKy+\nuFIzMysCtx/NzCwfPPrRzMys7rhSMzMrArcfzcwsF4Tbj2ZmZvXGlZqZWe7J7UczM8sRtx/NzMzq\niys1M7MicPvRzMzywSdfm5mZ1R1XamZmeedbz5iZWa64/WhmZlZfXKmZmeVecQaKOKmZmRVBh2Ic\nUytG6jYzs0JwpWZmlne+Sr+ZmVn9caVmZlYEPk/NzMzyoTijH4vxLs3MrBBcqZmZFYHbj2Zmlhtu\nP5qZmdUXV2pmZnknuf1oZmY54vajmZlZfXFSKxhJUyRt28y8bSVNb2Hd0ZLOabPgzKztNLYgK/Go\nYU5qOSLpVUk7NJl2iKQHG19HxHoRcV+7B9eCpjHWOklflzRJ0vuS3pF0k6R+Za47SFJI+qjk8XQF\nYjpb0tVLup1KkbS2pOslzZY0R9Izkk6W1LGN97vIL16Sjpc0QdJnkka3ZTy1I518XalHDavt6Mza\ngTKt+X/hWeCbQG9gFeAF4JJW7rZXRHRPjw1buW7FSarY8XVJawCPAa8DG0RET2BvYAjQo1L7WQJv\nAOcAf6p2IFZ5TmoFU1rNSeqavtm+J+lZYNMmy24s6QlJH0q6DujSZP6ukp5KFcvDkr7aZD+npG/o\ncyRdJ+lL65cZ76GSnksxvCzpqJJ5kyXtVvJ6qVQZbJxeD0txvS/p6dK2q6T7JI2S9BAwF1g9VYwv\np329Imn/hcUUEW9FxOsREWlSA7Bma99bM+/3sPR+35N0l6RVS+ZdKOl1SR9Imihp6zR9OPBjYJ/S\nyq9p5V5azZVUjIdLeg343zI+s7I+H+BnwMMRcXJEzEyf2dSI2D8i3k/b2j21wt9P/xbrluwnJK1Z\n8vqL6kupRS7pB5LeljRT0qFp3pHA/sAP0+dw68KCi4i/RcTNwDtl/aPkhduPVgA/BdZIj52Agxtn\nSOoM3AxcBfQBrge+XTJ/Y7JvukcBywF/AMZJWrpk+yOA4cBqwFeBQxYjxreBXYFlgUOBX0vaJM27\nEjigZNlvAjMj4snUDryN7Bt5H+AU4EZJK5QsfyBwJFn1MAu4CNg5InoAWwBPpfc6MP3xHVjy/gdK\neh/4JG37vMV4b18iaQ+y5LQXsALwAHBNySLjgY3S+/krcL2kLhFxJ/Bz4LrFqPy+BqwL7NTSZyap\nG818PguxA3BDC+9z7fS+Tkzv83bg1vQ7V46VgJ5AP+Bw4PeSekfEZcBfgPPS57Bb2t/Fki4uc9v5\n1HjrGbcfrQ7dnP4Av5/+6Lb0P/MIYFREvBsRr5P90Wo0DFgK+E1EzIuIG8j+qDY6EvhDRDwWEQ0R\nMQb4LK3X6KKIeCMi3gVuJfuD3CoRcVtEvBSZ/wPuBrZOs68Gvilp2fT6QLIkDFmyuz0ibo+IBRFx\nDzCBLPE1Gh0RUyJiPjAfWACsL6lrRMyMiCkphtcioldEvFYS12sR0QtYHjgT+Fcr39rskn+nU9K0\no4FfRMRzKaafAxs1VmsRcXVEvBMR8yPiV8DSwFdaud+mzo6IjyPiExb9mS3081mI5YCZLexzH+C2\niLgnIuYB5wNdyRJlOeYB/5V+L28HPqKFzyEijo2IY8vcttU5J7X82TP9Ae6V/ui29D/zKmTHPRpN\nazJvRkmLren8VYEfNEmgA9J6jd4seT4X6N6aNwIgaWdJj0p6N+3jm2SJhIh4A3gI+LakXsDOZN/U\nG+Pbu0l8WwErl2z+i/ceER+T/bE9Gpgp6TZJ6ywqvpSwxwC3qHXHpZYv+Xc6vyTmC0vifZfsO3a/\n9FmcklqTc9L8no2fxRIo/fdv9jNr5efzDl/+nJtahZLfpYhYkOIoa7AN8E5K+o0W63erWDxQxIph\nJlkiajSwybx+0pca6KXzXyer8nqVPJaJiNJ22RJJrcwbyb7J901J+nayP/SNxpBVGHsDj0TEjJL4\nrmoSX7eIOLdk3dKETUTcFRHfIPuD/C/g8jJD7QSsSNYiXRKvA0c1iblrRDycjp/9kKy67p0+izn8\n+7OIhWzvY2CZktcrLWSZ0vVa/Mxa8fn8g5JW9UK8QZZAgWygDtnvYeO/3dwy4m7Owj4HAx9Ts0IY\nC/xIUm9J/YHvlcx7hKwl931lAzD2AjYrmX85cLSkzZXpJmkXSYs7uk2SupQ+gM5kLbZZwHxJOwM7\nNlnvZmAT4ASyY2yNrgZ2k7STpI5pm9um97mwnfeVtEc6dvQZWUtrQTPL7iXpK5I6pGN0FwBPpqqt\ncUDGfYvxGVxK9u+xXtpOT0l7p3k9yP49ZgGdJP2ELyfRt4BB+vIozqeAkenfbyjwnUXsv9nPrDWf\nD9mx2i0k/VLSSum9rCnp6lRRjwV2kbS9pKWAH6RtPlwS934phuFkx/3K9RaweksLSOqUfr86Ao3v\n01dXygkntWL7GVkb6BWyY1WNx6OIiM/JBiwcQtYG2wf4W8n8CcARwO+A94AXWbyBII22IBt00fTx\nfbI/gu8B+wHjSldKx4JuJBuMUhrf60DjwItZZFXIqTT/O98BOJmsiniX7A/pMfDFoJCPSgaK9APu\nBD4EJpH9cf9WybYGkLVFWyUibgL+B7hW0gfAZLKWKsBdaZ/Pk/2bfcqXW4fXp5/vSHoiPT+LbBDQ\ne2T/1n9dxP5b+sya/XwWsp2XgP8HDAKmSJpD9m80AfgwIqaSVde/BWYDuwG7pd85yL6g7Aa8Tzaa\n8eaW4m7iCmBwap/eDCDpUkmXlixzJtnv1ukpjk/StHwrSPtRXz5kYlZ/UtWydkQcsMiF24Gkp4Dt\nI6JYQ8atZnXotWosve0ZFdvep7ccNTEihlZsgxXkktvqmqQ+ZMO6D6x2LI0iotWjPM2sMmq7jjRr\ngaQjyFpkd0TE/dWOx6xmqTijH12pWd2KiMspf4SiWbHV+KjFSqntlGtmZtYKrtTMzApABanUnNQW\nU68+y8Uq/QYuekGzEl07t+mdVyyHpk17ldmzZy9RRhJOarYIq/QbyJXj7qt2GFZn1h/Qs9ohWJ3Z\ncvOaHDlfs5zUzMzyTnz54nI55qRmZpZ7Kkz70aMfzcwsN1ypmZkVQFEqNSc1M7MCKEpSc/vRzMxy\nw5WamVkBFKVSc1IzM8u7Ag3pd/vRzMxyw5WamVnOyeepmZlZnkiq2KOMfZ0kaYqkyZKukdRFUh9J\n90h6If3sXbL8jyS9KGmqpJ1Kpg+RNCnNu0hl7NxJzczMKkZSP+D7wNCIWB/oCIwETgfujYi1gHvT\nayQNTvPXA4YDF0tqvPL3JcARwFrpMXxR+3dSMzMrgPas1MgObXWV1AlYBngD2AMYk+aPAfZMz/cA\nro2IzyLiFeBFYDNJKwPLRsSjERHAlSXrtLhjMzPLufY6phYRMySdD7wGfALcHRF3S+obETPTYm8C\nfdPzfsCjJZuYnqbNS8+bTm+RKzUzM2ut5SVNKHkc2TgjHSvbA1gNWAXoJumA0pVT5RVtEZgrNTOz\nvKv8eWqzI6K5G73tALwSEbMAJP0N2AJ4S9LKETEztRbfTsvPAAaUrN8/TZuRnjed3iJXamZmBdCO\nx9ReA4ZJWiaNVtweeA4YBxycljkYuCU9HweMlLS0pNXIBoQ8nlqVH0galrZzUMk6zXKlZmZmFRMR\nj0m6AXgCmA88CVwGdAfGSjocmAaMSMtPkTQWeDYtf1xENKTNHQuMBroCd6RHi5zUzMxyrr1Pvo6I\nnwI/bTL5M7KqbWHLjwJGLWT6BGD91uzbSc3MrAB8RREzM7M640rNzKwIilGoOamZmeWe3H40MzOr\nO67UzMwKoCiVmpOamVkBFCWpuf1oZma54UrNzCzninTnayc1M7MiKEZOc/vRzMzyw5WamVneFeg8\nNSc1M7MCKEpSc/vRzMxyw5WamVkBFKVSc1IzMyuCYuQ0tx/NzCw/XKmZmRWA249mZpYLUnGuKOL2\no5mZ5YYrNTOzAihKpeakZmZWAEVJam4/mplZbrhSMzMrgmIUak5qZmZF4PajmZlZnXGlZmaWd771\njJmZ5YWAguQ0tx/NzCw/XKmZmeVecS6T5aRmZlYABclpbj+amVl+uFIzMyuAorQfXamZmVluuFIz\nM8s7FeeYmpOamVnOCejQoRhZze1HMzPLDVdqZmYF4PajmZnlhkc/mpmZ1RlXamZmeefRj2ZmlhfZ\nVfqLkdXcfrQWvfnGdI7eb1dG7Lg5I3YaxjV/vgSAf9x+MyN2GsZma/Tm2Wee/GL5N6ZPY6t1V2K/\nXbZiv1224hdnnPTFvIvP/2922XI9tlm/X7u/D6tNR333MAausiJDNlq/2qFYTrhSsxZ16tSJE398\nDuusvxEff/QhB+2+LZtv9XXWWHtdzrvkKn5xxon/sU6/VVfjr7c9+B/Tt95+OCMOOoK9thvSHqFb\nHTjw4EM4+tjj+e5hB1U7lJzzVfrNAFh+xZVYfsWVAOjWvQeD1lybWW/OZPOtv97qbW2w8aaVDs/q\n3FZbb8O0V1+tdhiFUJCc5vajle+N6dOYOmUS623UcqX1xuvT2G+XrThy5Dd58vGH2yk6MzNXalam\nuR9/xGnHHsTJZ/2c7j2WbXa55VdYiVsfnEyv3n14btJTnHL0/lx35yMtrmNmba8o7cc2q9Qktfor\nuqRXJd1Y8vo7kkZXNLBFx3C2pFPac5+1bv68eZx27EEM331vthu+e4vLdl56aXr17gPAuhtsRP+B\ng3jtlZfaI0wza04a0l+pRy1rs6QWEVss5qpDJA1enBUlufKssIjgv08/nkFrrM3+3z1+kcu/985s\nGhoaAJj+2qu8/urL9Bs4qI2jNDPLtGWl9lH6ubKk+yU9JWmypK0XseqvgDMWsr0+km6W9IykRyV9\nNU0/W9JVkh4CrpJ0SFrunlT5HS/pZElPpvX6pPWOkDRe0tOSbpS0TIU/glx4esKj3H7TdUx45P4v\nhuk/9M+7+eddt7LLFoOZ9OR4Tjp8BN87eC8Annz8Ifb95pbst8tWnH7cQZx+zgX07NUbgIvO/Qm7\nbDGYTz+Zyy5bDOay3/yimm/NasBBB+zLtlv/P56fOpU1BvVn9J+uqHZIudR4nlqlHrVMEdE2G5Y+\niojukn4AdImIUZI6AstExIfNrPMqsDlwH7AbsBGwa0QcIum3wOyI+Jmk7YALImIjSWenZbeKiE8k\nHQKcCWwMdAFeBE6LiEsl/RqYFhG/kbRcRLyT9nsO8FZE/DZt76OIOH8h8R0JHAmw0ioDhtz64KSK\nfFZWHOsP6FntEKzObLn5UCZOnLBEmaRbv6/EusdcWqmQmHjWdhMjYmjFNlhB7TH6cTxwaEoWGzSX\n0Eo0AL8EftRk+lbAVQAR8b/AcpIaRx+Mi4hPSpb9Z0R8GBGzgDnArWn6JGBQer6+pAckTQL2B9Zb\n1BuJiMsiYmhEDO3dZ7lFLW5mZu2szZNaRNwPbAPMAEZLKucsy6vSOgPK3M3HTV5/VvJ8QcnrBfx7\nxOdo4PiI2AD4GVlVZ2aWS0VpP7Z5UpO0Kllr73Lgj8Ami1onIuYBvwZOKpn8AFlFhaRtyVqRHyxB\naD2AmZKWatyumVleFWX0Y3uMFtwWOFXSPOAjoNzr4VxBdmys0dnAnyQ9A8wFDl7CuM4CHgNmpZ89\nlnB7ZmZWZW2W1CKie/o5BhhT5jqDSp5/BqxS8vpdYM+FrHN2k9ejyVqLC9vmF/Mi4hLgkkVtz8ys\n7qk4J1/7vC4zs5zLhvRXO4r2UZWkJukxYOkmkw+MCI+RNzOzxVaVpBYRm1djv2ZmxVT7oxYrxe1H\nM7MCKEhO861nzMwsP1ypmZkVgNuPZmaWD3Vw0nSluP1oZma54UrNzCznGm89UwROamZmBVCUpOb2\no5mZ5YYrNTOzAihIoeakZmZWBG4/mpmZ1RlXamZmeefz1MzMLC+ULmhcqUdZ+5R6SbpB0r8kPSfp\n/0nqI+keSS+kn71Llv+RpBclTZW0U8n0IZImpXkXaREBOKmZmVlbuBC4MyLWATYEngNOB+6NiLWA\ne9NrJA0GRgLrAcOBiyV1TNu5BDgCWCs9hre0Uyc1M7MCkCr3WPS+1BPYBrgCICI+j4j3gT2AMWmx\nMcCe6fkewLUR8VlEvAK8CGwmaWVg2Yh4NCICuLJknYXyMTUzswLo0L4H1VYDZgF/lrQhMBE4Aegb\nETPTMm8CfdPzfsCjJetPT9PmpedNpzfLlZqZmbXW8pImlDyObDK/E7AJcElEbAx8TGo1NkqVV1Q6\nMFdqZmYFUOFCbXZEDG1h/nRgekQ8ll7fQJbU3pK0ckTMTK3Ft9P8GcCAkvX7p2kz0vOm05vlSs3M\nLOeyY2HtN/oxIt4EXpf0lTRpe+BZYBxwcJp2MHBLej4OGClpaUmrkQ0IeTy1Kj+QNCyNejyoZJ2F\ncqVmZmZt4XvAXyR1Bl4GDiUrpMZKOhyYBowAiIgpksaSJb75wHER0ZC2cywwGugK3JEezXJSMzMr\ngA7tfPJ1RDwFLKxFuX0zy48CRi1k+gRg/XL366RmZlYAvvajmZlZnXGlZmZWAAUp1JzUzMzyTmTX\nfywCtx/NzCw3XKmZmRVAe49+rBYnNTOzvGvFLWPqnduPZmaWG67UzMwKoCCFmpOamVneiXa/9UzV\nuP1oZma54UrNzKwAClKoOamZmRWBRz+amZnVGVdqZmY5l90ktNpRtA8nNTOzAvDoRzMzszrTbKUm\nadmWVoyIDyofjpmZtYVi1Gkttx+nAMGXP4vG1wEMbMO4zMysgooy+rHZpBYRA9ozEDMzsyVV1jE1\nSSMl/Tg97y9pSNuGZWZmlZJdJqtyj1q2yKQm6XfA14ED06S5wKVtGZSZmVVQuvVMpR61rJwh/VtE\nxCaSngSIiHcldW7juMzMzFqtnKQ2T1IHssEhSFoOWNCmUZmZWUXVeIFVMeUktd8DNwIrSPoZMAL4\nWZtGZWZmFVXrbcNKWWRSi4grJU0EdkiT9o6IyW0blpmZWeuVe5msjsA8shakr0JiZlZHGkc/FkE5\nox/PAK4BVgH6A3+V9KO2DszMzCrHox//7SBg44iYCyBpFPAk8Iu2DMzMzKy1yklqM5ss1ylNMzOz\nOlHb9VXltHRB41+THUN7F5gi6a70ekdgfPuEZ2ZmS0oqzq1nWqrUGkc4TgFuK5n+aNuFY2Zmtvha\nuqDxFe0ZiJmZtZ2CFGqLPqYmaQ1gFDAY6NI4PSLWbsO4zMzMWq2cc85GA38mO864MzAWuK4NYzIz\nsworypD+cpLaMhFxF0BEvBQRZ5IlNzMzqxNS5R61rJwh/Z+lCxq/JOloYAbQo23DMjMza71yktpJ\nQDfg+2TH1noCh7VlUGZmVjlCHtLfKCIeS08/5N83CjUzs3pRB23DSmnp5OubSPdQW5iI2KtNIjIz\nM1tMLVVqv2u3KOpQ184dWX9Az2qHYXWm96bHVzsEqzOfTX2tItup9VGLldLSydf3tmcgZmbWdopy\nz7CivE8zMyuAcm8SamZmdUq4/fgfJC0dEZ+1ZTBmZtY2fOfrRNJmkiYBL6TXG0r6bZtHZmZm1krl\nHFO7CNgVeAcgIp4Gvt6WQZmZWWV1UOUetayc9mOHiJjWpB/b0EbxmJlZhWXXbKzxbFQh5SS11yVt\nBoSkjsD3gOfbNiwzM7PWKyepHUPWghwIvAX8I00zM7M6Uettw0op59qPbwMj2yEWMzNrIwXpPpZ1\n5+vLWcg1ICPiyDaJyMzMbDGV0378R8nzLsC3gNfbJhwzM6s0gW890ygirit9Lekq4ME2i8jMzCqu\nKNdEXJz3uRrQt9KBmJmZLalyjqm9x7+PqXUA3gVOb8ugzMyssgrSfWw5qSk7W29DYEaatCAimr1x\nqJmZ1R5JhTmm1mL7MSWw2yOiIT2c0MzMrGaVc0ztKUkbt3kkZmbWZrJLZVXmUcuabT9K6hQR84GN\ngfGSXgI+JhsdGhGxSTvFaGZmS8hXFIHHgU2A3dspFjMzsyXSUlITQES81E6xmJlZG/DJ15kVJJ3c\n3MyIuKAN4jEzszZQkJzWYlLrCHQnVWxmZma1rqWkNjMi/qvdIjEzs7ZRB3esrpRFHlMzM7P6p4L8\nSW/pPLXt2y0KMzOzCmi2UouId9szEDMzaxvZ6MdqR9E+yrmfmpmZ1bmiJLWi3GLHzMwKwJWamVkB\nqCAnqrlSMzPLucZjapV6lLVPqaOkJyX9Pb3uI+keSS+kn71Llv2RpBclTZW0U8n0IZImpXkXqYzM\n7KRmZmZt4QTguZLXpwP3RsRawL3pNZIGAyOB9YDhwMWSOqZ1LgGOANZKj+GL2qmTmplZ3lXwtjPl\ndDEl9Qd2Af5YMnkPYEx6PgbYs2T6tRHxWUS8ArwIbCZpZWDZiHg03cvzypJ1muVjamZmBVDhCxov\nL2lCyevLIuKykte/AX4I9CiZ1jciZqbnbwJ90/N+wKMly01P0+al502nt8hJzczMWmt2RAxd2AxJ\nuwJvR8RESdsubJmICEnRFoE5qZmZ5Vw7n3y9JbC7pG8CXYBlJV0NvCVp5YiYmVqLb6flZwADStbv\nn6bNSM+bTm+Rj6mZmRVAex1Ti4gfRUT/iBhENgDkfyPiAGAccHBa7GDglvR8HDBS0tKSViMbEPJ4\nalV+IGlYGvV4UMk6zXKlZmZm7eFcYKykw4FpwAiAiJgiaSzwLDAfOC4iGtI6xwKjga7AHenRIic1\nM7PcEx2qcJX+iLgPuC89f4dmLpQfEaOAUQuZPgFYvzX7dFIzM8s5UZw7X/uYmpmZ5YYrNTOzvPOd\nr83MLE8qfPJ1zXL70czMcsOVmplZzhVpoIiTmplZAbj9aGZmVmdcqZmZFUBBCjUnNTOzvBPFacsV\n5X2amVkBuFIzM8s7gQrSf3RSMzMrgGKkNLcfzcwsR1ypmZnlXHbn62LUak5qZmYFUIyU5vajmZnl\niCs1M7MCKEj30UnNzCz/VJgh/W4/mplZbrhSMzPLuSJdJstJzcysANx+NDMzqzOu1MzMCqAYdZqT\nmlXQ3XfdySknn0BDQwOHHPZdTv3h6dUOyarouH235dC9tkASf/7bQ/zur/cBcMzIr3HUiK1pWBDc\n+cBkzrjwFpbq1JHfnbkvmwweyIJYwCnn3cgDE18AYMTwIZx62E5EBDNnzeGwM8fwzvsfV++N1SNf\n0NisdRoaGjjx+8dx2x330K9/f7Yatim77ro76w4eXO3QrAoGr7Eyh+61BVsf+Es+n9fAuN8fy+0P\nTKZ/397suu0GbLbPuXw+bz4r9O4OwGF7bQnApiN+zgq9u3Pz745lqwN+SYcO4penfodNvn0O77z/\nMaNO2IOj9/kao/5wezXfntUwH1Ozihj/+OOsscaarLb66nTu3Jm99xnJ32+9pdphWZWss9pKjJ/8\nKp98Oo+GhgU8MPFF9txuI47ce2vO//M9fD5vPgCz3vsoW371lbhv/NQvps358BOGDB6IlJ003K1r\nZwB6dO/KzFlzqvOm6ljj6MdKPWpZrcdndeKNN2bQv/+AL17369efGTNmVDEiq6YpL73BlhuvSZ+e\n3ejaZSmGb7Ue/VfqzZqrrsiWG6/B/Veewt1/PIEhgwcCMOn5Gez6tQ3o2LEDq66yHBsPHkD/lXoz\nf/4CTvj5dYwf+2NevnsU666+EqNvfrjK764+SarYo5a5/WhmFTf1lbf41eh7uPXi45j76ec8PXU6\nDQ0L6NSxA316dmObg85n6HqrcvV5h7Hurmcz5pZHWGe1vjz0lx/y2sx3efTpV7LlO3XgiO9szbB9\n/4dXps/m16ftzamH7cj//PGuar9Fq1HtVqlJWqyvV5I2khSShpdM6yXp2JLXgyTttwSx3Sdp6OKu\nb7DKKv2YPv31L17PmDGdfv36VTEiq7YxNz/ClvufxzcO/w3vfzCXF6a9zYy33ufme58CYMKUaSxY\nECzfuzsNDQv44a/+xrCR5zLipMvo1aMrL7z2Nhuu3R+AV6bPBuCGe55g2IarV+091TNV8FHL2i2p\nRcQWi7nqvsCD6WejXsCxJa8HAYud1GzJDd10U1588QVefeUVPv/8c66/7lp22XX3aodlVdQ4CGTA\nSr3ZY7sNue6OCdx63zN8bdO1AVhz4Ip0XqoTs9/7iK5dlmKZLtlxs+02X4f5DQv418tv8sasOayz\n+kosn7a1/bB1mPrKm9V5Q1YX2q39KOmjiOguaWXgOmDZtP9jIuKBZtYRsDfwDeABSV0i4lPgXGAN\nSU8B9wBbA+um12OAm4CrgG5pU8dHxMNpm6cBBwALgDsi4vSS/XUA/gRMj4gzFxLPkcCRAAMGDlyi\nzyNvOnXqxK8v/B277bITDQ0NHHzIYQxeb71qh2VVdM3536VPr27Mm9/AieeOZc5HnzDm5kf4w9n7\nM+H6H/P5vAa++5OrAFihdw9uvfg4FiwI3pj1PoefOQaAmbPm8PPL7uCeP57IvPkNvDbzXY786dXV\nfFt1q8YPhVWMIqJ9dvTvpPYDoEtEjJLUEVgmIj5sZp0tgf+KiO0l/RW4MSJulDQI+HtErJ+W2xY4\nJSJ2Ta+XARZExKeS1gKuiYihknYGzgJ2iIi5kvpExLuS7gNOB04AJkfEqEW9nyFDhsZDj01Yos/E\niqf3psdXOwSrM59NHcuCuW8vUUpaa70N44Jr765USOz+1ZUmRkRNHrKpxujH8cChks4GNmguoSX7\nAtem59fFMT/6AAARfUlEQVTy5RZkS5YCLpc0CbgeaDxZagfgzxExFyAi3i1Z5w+UmdDMzKw2tXtS\ni4j7gW2AGcBoSQctbLlUxX0b+ImkV4HfAsMl9ShjNycBbwEbAkOBzmWs8zDwdUldyljWzKyuNJ7z\nV4lHLWv3pCZpVeCtiLgc+COwSTOLbg88ExEDImJQRKwK3Ah8C/gQKE1uTV/3BGZGxALgQKBjmn4P\nWZW4TIqlT8k6VwC3A2Ml+VQHM8sRVfS/WlaN9uO2wNOSngT2AS5sZrl9yQZ8lLoR2Dci3gEekjRZ\n0i+BZ4AGSU9LOgm4GDhY0tPAOsDHABFxJzAOmJAGlZxSuvGIuAB4ErgqDRoxM7M60m4VSUR0Tz/H\nkI1QXNTyhy5k2jiypERENB3Cv12T118teX5ayTbOJRs9WbrdbUue/3RRsZmZ1ZtabxtWittsZmY5\nl137sRhZrSaSmqTHgKWbTD4wIiZVIx4zM6tPNZHUImLzasdgZpZbdTBqsVJqIqmZmVnbKkpS8wg/\nMzPLDVdqZmYFUOvnl1WKk5qZWc4J6FCMnOb2o5mZ5YcrNTOzAnD70czMcsOjH83MzOqMKzUzswJw\n+9HMzHLBox/NzMzqkCs1M7Pcq/2be1aKk5qZWd4V6ILGbj+amVluuFIzMyuAghRqTmpmZnmXjX4s\nRlpz+9HMzHLDlZqZWQEUo05zUjMzK4aCZDW3H83MLDdcqZmZFYBPvjYzs9woyOBHtx/NzCw/XKmZ\nmRVAQQo1JzUzs0IoSFZz+9HMzHLDlZqZWc4Jj340M7O88K1nzMzM6o+TmplZAaiCj0XuSxog6Z+S\nnpU0RdIJaXofSfdIeiH97F2yzo8kvShpqqSdSqYPkTQpzbtIarnmdFIzMyuC9sxqMB/4QUQMBoYB\nx0kaDJwO3BsRawH3ptekeSOB9YDhwMWSOqZtXQIcAayVHsNb2rGTmpmZVVREzIyIJ9LzD4HngH7A\nHsCYtNgYYM/0fA/g2oj4LCJeAV4ENpO0MrBsRDwaEQFcWbLOQnmgiJlZ7qlqox8lDQI2Bh4D+kbE\nzDTrTaBvet4PeLRktelp2rz0vOn0ZjmpmZkVQIVHPy4vaULJ68si4rL/3Ke6AzcCJ0bEB6WHwyIi\nJEVFo8JJzczMWm92RAxtaQFJS5EltL9ExN/S5LckrRwRM1Nr8e00fQYwoGT1/mnajPS86fRm+Zia\nmVnOVXKMSJmjHwVcATwXEReUzBoHHJyeHwzcUjJ9pKSlJa1GNiDk8dSq/EDSsLTNg0rWWShXamZm\nRdC+h9S2BA4EJkl6Kk37MXAuMFbS4cA0YARAREyRNBZ4lmzk5HER0ZDWOxYYDXQF7kiPZjmpmZlZ\nRUXEgzSfRrdvZp1RwKiFTJ8ArF/uvp3UzMwKwNd+NDOz3PC1H83MzOqMKzUzswIoSKHmpGZmlnvl\nX7Ox7rn9aGZmueFKzcysADz60czMckF49KOZmVndcaVmZlYABSnUnNTMzAqhIFnN7UczM8sNV2pm\nZgXg0Y9mZpYbHv1oZmZWZ1ypmZkVQEEKNSc1M7NCKEhWc/vRzMxyw5WamVnOZRfpL0ap5qRmZpZ3\n8uhHMzOzuuNKzcysAApSqDmpmZkVQkGympPaYnriiYmzuy6ladWOowYtD8yudhBWd/x707xVqx1A\nPXFSW0wRsUK1Y6hFkiZExNBqx2H1xb83bU0e/WhmZvnh0Y9mZmZ1xpWaVdpl1Q7A6pJ/b9qQKMw4\nESc1q6yI8B8nazX/3rSDgmQ1tx/NzCw3XKmZmRVAUUY/ulIzM7PccKVmVSepD7B8RDxf7Vis/khS\nRES146h1HtJv1g4kdQG+Dxwmad1qx2P1Q9IAACe08qiCj1rmpGZVFRGfAv9IL/eWNLia8VjtktRd\nUuf0fF3gPEk9qhyW1RgnNasaKWuIRMSDwDhgWeA7TmzWlKRuwF+AvdOkuenxkaSl0jK1XkRUT7qf\nWqUetcxJzaqi8TiIpNUkdYqIh4E/Az3JEptbkfaFiPgYuA44VNI+wCDgk8jMS8u4DdmiYjQgPVDE\nqiIltF2As4AHJH0E/IbsyhKHAwdI+ktEPFvNOK36JHWMiIaI+KukWcBpwERgNUkXAtOBz4BOEXFB\nNWO16nOlZlUhaRjwc2Afsi9XewLnAbOAMUA34POqBWg1IVX0DZK+Iem8iLgHuBDYnuz347X0szvw\nWBVDrWmiOO1HV2rWriR1AILs/lkHAesA2wCnA0cC55N9Ez8jtZyswFJFvz1wMXBUmnarpPnAycDz\nEXFrNWOsFzWeiyrGlZq1i5KD+N3TcZC/R8TTZBXadyPiLuBtsi9afZ3QTJlOwHDgrIj438bRjxFx\nB3ApcJqkftWM02qLk5q1i5JjaPdKOlvSXmnWisCRkjYHNgPOj4jJVQvUakb68jMf+BQYJqlLRHwO\nIGlT4HZg94iYUc0460VR2o9OatYuJK0M7E/WXnwX2CklucOAAcBPgF9ExDPVi9KqrbGilzRQUv80\n+Q5gKeBrad6GwK+BtSPi3aoEWodUwf9qmY+pWZuTNBTYEJgREddJWgHYCfgWsFRE7CppmYiY60se\nFVtJRf8L4GFJfSJiRDrF40BJp5Gd9nFOal+bfYmTmrUpSduSjWa8i2yY/jUR8YSkO4DOwB6SHo+I\nN8DnGhVVyXmLw8hGwe5KVpn9SdI/ImIHSaPJvhzNiYiX/AWolWq7wKoYJzVrM5JWA34MHBgR90t6\nEbha0v4R8aSkW4A7GxOaFU+69ue8NGy/L/AOMAJYi2y0Y0/gPkkPR8QWwBON6zqhtU5BcpqPqVll\nlRwT2ZTsm3ZPshGORMR5wBXAOElDIuIdJ7TiSqd3bAGcKGlXsuOqHwLPArsAf4qID8kq/YHpd8qs\nRU5qVlGphbQNWQtpEtkJ1stIOj7N/xXwe7KTZc2eAXYErgJuiIg3yYqKmcAako4ga0V+IyLGVy/M\n+lbJkY8e/WiFIukrwDHA6IiYCNwH3AusI+kHABFxbkT8ny9AW0ySuknqHxELgFXT5H8CO6dh+wvI\n7twwlyyhXRoRz1Up3NwoyuhHJzWrtA2AvsAOklaIiDnAncDDwFckNf4R8zGR4hoE/FbSGcApwA+A\n75HdpaHx2o0vkyW6b0fE3/wFyMrlpGZLpOQYWn9JPSPiBrKLFH9AdrX95dJxkVuBn0TEtCqGazUg\nIqYAL5INInosnWw/i+xSWEtLupeswp+XTr72F6BKKMZF+j360RafpA4RsUDSzmTH0KZKWpFsYMjf\ngZ3Jzi26KiLeIRsEYAUkqRfweUTMTZMmA78CDpI0KSLuBZ5J1ds3gDci4tEqhZtLNZ6LKsZJzVpN\nUteI+CQltDWB/waOioiHJV0E3Ex2cvVS6Wc3sqHaVkCS+gDPA/+Q9EBE/D4ixqR5rwMXSDoYeB/Y\nq/H2MT4PzRaHk5q1iqSewLmSboqIu8n+EP2L7I8WEfF9SdcAp0fETyWNj4iZVQzZqu894G6yEY37\nS9oMeBC4PiIul/Q5cCMwHzixcSUntMoqylFJH1Oz1lqW7HjIfumWIB8AywE7lCxzO+leaE5olpLT\nE2QDiLYBRqef/yfp62QDQjYnGxRyR7XizLdKjn2s7ezoSs3KIqlHRHwYEa9LuhIYSXYx4llkB/xH\nS1oHmJOm/7B60VqtiYjzJd1O9uVnMrARWYU/ElgT2Md3Z7BKcFKzRZI0CLhB0kRgLPAC8GfgM7Lh\n2P8D7E02MGQV4KSI+IePiRiApI4R0UBWoX2L7Ar7V6REtyLZRa1nVzPGvGu883UROKlZOboAKwN7\nAK+SXRHkUqA32flnZwGjIuLC0pWc0AwgJTSAx4CzgUci4vw0bZZ/T6ySfEzNWpSG7f+LrG00B3gN\n2Ad4g+zajt9Jr8+T1Ctdz8/sS1LVPg04GejeeLdqJzSrNFdq1qI0bL9DRDwn6QDgWuDnEXGFpBvI\nrqa+B/BURLxf1WCtqkpuH9MhXerqCyXJazqw4D/Xtrbm9qNZUpLYxksaCVyTrtH3e2Aq2YnXPq+o\nwEoS2vZkldhdEfFp0+UiYrKk0yJiRhXCLLRaH7VYKW4VWVlKExtZu/EsScc1WcYJrYDSQJCQNBy4\nBHhvYQlNmQ4RMU3SMpKWa/9oLe+c1OxLSq7l+B+/GyWJbSKwGzClveOz2iFpzXSqR4Ok3mQDho5O\nN4TdWtLB6UTrRo2XVetFdm5an6oEXkQFuvWM24/2hXJaSE0qNrcci60vsKKkRyPiPUn/BA5P90Dr\nAMwjO+b6uKROETE/XZHmeuDUiHiheqEXSx1ch7hiXKkZUH4LqXHxtE5XsmH9VkAR8RDZjWBflrQs\n2XlojwO/jYh9yM5pXE9S55TQegM3Af8VEfdXK27LNye1gmttC6nxRNrUQrqP7BJZVlDptkInkJ2v\nODsiLkwXtt6a7ELXf4yIz9Pi+wLnRMQDVQq32HzrGSsIt5BsiUTELZLmARMlDQE+JTt/8cyIuK2x\nRR0RF1c30mIryuhHJ7WCi4iHJPUgayF9layFtAswPn3j3h04NLWQPk/V3I3AT/2N2xpFxO2SFgDP\nAV8BTouIT0uO0/rYq7ULtx/NLSSriIi4E/gusHHj8djGROaEVn0e/WiF4haSVUJE3AYeFVuLajwX\nVYyTmn3BLSSrFP+eWLW4/Whf4haSWU614+hHScMlTZX0oqTTK/1WWuJKzf6DW0hm+dNeox8ldSS7\nPdU3yC5gPV7SuIh4tj3270rNmuWEZmaLYTPgxYh4OQ0wu5bsTh7twpWamVnOtfOdr/sBr5e8ng5s\n3l47d1IzM8u5J56YeFfXpbR8BTfZRdKEkteXRcRlFdz+YnNSMzPLuYgY3o67mwEMKHndP01rFz6m\nZrklqUHSU5ImS7pe0jJLsK1tJf09Pd+9pRFdknpJOnYx9nG2pFPKnd5kmdGSvtOKfQ2SNLm1MZqV\nYTywlqTVJHUGRgLj2mvnTmqWZ59ExEYRsT7wOXB06czGm1a2dqMRMS4izm1hkV5Aq5OaWR5ExHzg\neOAusnNex0ZEu9170UnNiuIBYM1UoUyVdCUwGRggaUdJj0h6IlV03eGLc23+JekJYK/GDUk6RNLv\n0vO+km6S9HR6bAGcC6yRqsRfpuVOlTRe0jOSflayrTMkPS/pQbIT3lsk6Yi0nacl3dik+txB0oS0\nvV3T8h0l/bJk30ct6QdptigRcXtErB0Ra0TEqPbct5Oa5Z6kTsDOZPf+guyuAxdHxHrAx8CZwA4R\nsQkwAThZUhfgcrI7fA8BVmpm8xcB/xcRGwKbkN0N/HTgpVQlnippx7TPzYCNgCGStkmXIxuZpn0T\n2LSMt/O3iNg07e854PCSeYPSPnYBLk3v4XBgTkRsmrZ/hKTVytiPWV3yQBHLs66SnkrPHwCuAFYB\npkXEo2n6MGAw8JCyMc+dgUeAdYBXGm+tI+lq4MiF7GM74CCAiGgA5qQ7GZTaMT2eTK+7kyW5HsBN\nETE37aOc4w7rSzqHrMXZnazF02hsRCwAXpD0cnoPOwJfLTne1jPt+/ky9mVWd5zULM8+iYiNSiek\nxPVx6STgnojYt8lyX1pvCQn4RUT8ock+TlyMbY0G9oyIpyUdAmxbMq/pyfKR9v29iChNfkgatBj7\nNqt5bj9a0T0KbClpTQBJ3SStDfwLGCRpjbTcvs2sfy9wTFq3o7IbqH5IVoU1ugs4rORYXT9JKwL3\nA3tK6qrsnna7lRFvD2CmpKWA/ZvM21tShxTz6sDUtO9j0vJIWltStzL2Y1aXXKlZoUXErFTxXCNp\n6TT5zIh4XtKRwG2S5pK1L3ssZBMnAJdJOhxoAI6JiEckPZSGzN+RjqutCzySKsWPgAMi4glJ1wFP\nA2+TDYVelLOAx4BZ6WdpTK8BjwPLAkenOyz8kexY2xPKdj4L2LO8T8es/siX9zMzs7xw+9HMzHLD\nSc3MzHLDSc3MzHLDSc3MzHLDSc3MzHLDSc3MzHLDSc3MzHLDSc3MzHLj/wNjaHVwRuWq8QAAAABJ\nRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fcb934e7e80>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot(actual_value = Train.actual_value_, pred_value = Train.pred_value_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-23T21:43:47.494886Z",
     "start_time": "2017-06-23T21:43:47.444773Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#4.5 GB\n",
    "pd.Series(Train.pred_value).to_csv('GRU_prediction_values.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-23T21:43:47.531782Z",
     "start_time": "2017-06-23T21:43:47.499601Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>train_score</th>\n",
       "      <th>test_score</th>\n",
       "      <th>test_score_20</th>\n",
       "      <th>time_taken</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_of_features</th>\n",
       "      <th>hidden_layers</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"35\" valign=\"top\">1</th>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.937278</td>\n",
       "      <td>0.880675</td>\n",
       "      <td>14.150092</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>12</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.957727</td>\n",
       "      <td>0.919578</td>\n",
       "      <td>25.316301</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>18</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.958215</td>\n",
       "      <td>0.920506</td>\n",
       "      <td>42.115427</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.999867</td>\n",
       "      <td>0.999747</td>\n",
       "      <td>39.416124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>12</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.999911</td>\n",
       "      <td>0.999831</td>\n",
       "      <td>47.097980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>0.997460</td>\n",
       "      <td>0.894029</td>\n",
       "      <td>0.798397</td>\n",
       "      <td>62.728601</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>12</td>\n",
       "      <td>0.999762</td>\n",
       "      <td>0.962473</td>\n",
       "      <td>0.928608</td>\n",
       "      <td>124.716464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>18</td>\n",
       "      <td>0.999921</td>\n",
       "      <td>0.972498</td>\n",
       "      <td>0.947679</td>\n",
       "      <td>186.515223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.937278</td>\n",
       "      <td>0.880675</td>\n",
       "      <td>14.150092</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>12</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.957727</td>\n",
       "      <td>0.919578</td>\n",
       "      <td>25.316301</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>18</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.958215</td>\n",
       "      <td>0.920506</td>\n",
       "      <td>42.115427</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.999867</td>\n",
       "      <td>0.999747</td>\n",
       "      <td>39.416124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>12</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.999911</td>\n",
       "      <td>0.999831</td>\n",
       "      <td>47.097980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>0.997460</td>\n",
       "      <td>0.894029</td>\n",
       "      <td>0.798397</td>\n",
       "      <td>62.728601</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>12</td>\n",
       "      <td>0.999762</td>\n",
       "      <td>0.962473</td>\n",
       "      <td>0.928608</td>\n",
       "      <td>124.716464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>18</td>\n",
       "      <td>0.999921</td>\n",
       "      <td>0.972498</td>\n",
       "      <td>0.947679</td>\n",
       "      <td>186.515223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>0.999841</td>\n",
       "      <td>0.937278</td>\n",
       "      <td>0.880675</td>\n",
       "      <td>14.001841</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.956175</td>\n",
       "      <td>0.916624</td>\n",
       "      <td>19.982509</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.999468</td>\n",
       "      <td>0.998987</td>\n",
       "      <td>33.803133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>0.979759</td>\n",
       "      <td>0.807399</td>\n",
       "      <td>0.635105</td>\n",
       "      <td>35.490781</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>12</td>\n",
       "      <td>0.997936</td>\n",
       "      <td>0.907825</td>\n",
       "      <td>0.826160</td>\n",
       "      <td>171.106554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>18</td>\n",
       "      <td>0.999762</td>\n",
       "      <td>0.965401</td>\n",
       "      <td>0.934262</td>\n",
       "      <td>255.321470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>0.999921</td>\n",
       "      <td>0.947303</td>\n",
       "      <td>0.899747</td>\n",
       "      <td>7.824117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>18</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.950231</td>\n",
       "      <td>0.905316</td>\n",
       "      <td>56.515639</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>0.999603</td>\n",
       "      <td>0.986027</td>\n",
       "      <td>0.973418</td>\n",
       "      <td>42.663989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>12</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.997782</td>\n",
       "      <td>0.995781</td>\n",
       "      <td>94.612861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>0.993332</td>\n",
       "      <td>0.786684</td>\n",
       "      <td>0.594262</td>\n",
       "      <td>85.770908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>12</td>\n",
       "      <td>0.994920</td>\n",
       "      <td>0.833925</td>\n",
       "      <td>0.684473</td>\n",
       "      <td>136.350251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>18</td>\n",
       "      <td>0.998571</td>\n",
       "      <td>0.921265</td>\n",
       "      <td>0.850464</td>\n",
       "      <td>256.658391</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>0.999206</td>\n",
       "      <td>0.958259</td>\n",
       "      <td>0.920591</td>\n",
       "      <td>19.687884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.999290</td>\n",
       "      <td>0.998650</td>\n",
       "      <td>33.079605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>0.974361</td>\n",
       "      <td>0.780829</td>\n",
       "      <td>0.584895</td>\n",
       "      <td>17.424911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>12</td>\n",
       "      <td>0.995634</td>\n",
       "      <td>0.797241</td>\n",
       "      <td>0.614346</td>\n",
       "      <td>173.522122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>18</td>\n",
       "      <td>0.999762</td>\n",
       "      <td>0.966288</td>\n",
       "      <td>0.935865</td>\n",
       "      <td>260.093316</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>73 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                              epoch  train_score  test_score  test_score_20  \\\n",
       "no_of_features hidden_layers                                                  \n",
       "1              1                  6     1.000000    0.937278       0.880675   \n",
       "               1                 12     1.000000    0.957727       0.919578   \n",
       "               1                 18     1.000000    0.958215       0.920506   \n",
       "               3                  6     1.000000    0.999867       0.999747   \n",
       "               3                 12     1.000000    0.999911       0.999831   \n",
       "               5                  6     0.997460    0.894029       0.798397   \n",
       "               5                 12     0.999762    0.962473       0.928608   \n",
       "               5                 18     0.999921    0.972498       0.947679   \n",
       "               1                  6     1.000000    0.937278       0.880675   \n",
       "               1                 12     1.000000    0.957727       0.919578   \n",
       "               1                 18     1.000000    0.958215       0.920506   \n",
       "               3                  6     1.000000    0.999867       0.999747   \n",
       "               3                 12     1.000000    0.999911       0.999831   \n",
       "               5                  6     0.997460    0.894029       0.798397   \n",
       "               5                 12     0.999762    0.962473       0.928608   \n",
       "               5                 18     0.999921    0.972498       0.947679   \n",
       "               1                  6     0.999841    0.937278       0.880675   \n",
       "...                             ...          ...         ...            ...   \n",
       "               1                  6     1.000000    0.956175       0.916624   \n",
       "               3                  6     1.000000    0.999468       0.998987   \n",
       "               5                  6     0.979759    0.807399       0.635105   \n",
       "               5                 12     0.997936    0.907825       0.826160   \n",
       "               5                 18     0.999762    0.965401       0.934262   \n",
       "               1                  6     0.999921    0.947303       0.899747   \n",
       "               1                 18     1.000000    0.950231       0.905316   \n",
       "               3                  6     0.999603    0.986027       0.973418   \n",
       "               3                 12     1.000000    0.997782       0.995781   \n",
       "               5                  6     0.993332    0.786684       0.594262   \n",
       "               5                 12     0.994920    0.833925       0.684473   \n",
       "               5                 18     0.998571    0.921265       0.850464   \n",
       "               1                  6     0.999206    0.958259       0.920591   \n",
       "               3                  6     1.000000    0.999290       0.998650   \n",
       "               5                  6     0.974361    0.780829       0.584895   \n",
       "               5                 12     0.995634    0.797241       0.614346   \n",
       "               5                 18     0.999762    0.966288       0.935865   \n",
       "\n",
       "                              time_taken  \n",
       "no_of_features hidden_layers              \n",
       "1              1               14.150092  \n",
       "               1               25.316301  \n",
       "               1               42.115427  \n",
       "               3               39.416124  \n",
       "               3               47.097980  \n",
       "               5               62.728601  \n",
       "               5              124.716464  \n",
       "               5              186.515223  \n",
       "               1               14.150092  \n",
       "               1               25.316301  \n",
       "               1               42.115427  \n",
       "               3               39.416124  \n",
       "               3               47.097980  \n",
       "               5               62.728601  \n",
       "               5              124.716464  \n",
       "               5              186.515223  \n",
       "               1               14.001841  \n",
       "...                                  ...  \n",
       "               1               19.982509  \n",
       "               3               33.803133  \n",
       "               5               35.490781  \n",
       "               5              171.106554  \n",
       "               5              255.321470  \n",
       "               1                7.824117  \n",
       "               1               56.515639  \n",
       "               3               42.663989  \n",
       "               3               94.612861  \n",
       "               5               85.770908  \n",
       "               5              136.350251  \n",
       "               5              256.658391  \n",
       "               1               19.687884  \n",
       "               3               33.079605  \n",
       "               5               17.424911  \n",
       "               5              173.522122  \n",
       "               5              260.093316  \n",
       "\n",
       "[73 rows x 5 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "past_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-23T21:43:47.551239Z",
     "start_time": "2017-06-23T21:43:47.533349Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>train_score</th>\n",
       "      <th>test_score</th>\n",
       "      <th>test_score_20</th>\n",
       "      <th>time_taken</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_of_features</th>\n",
       "      <th>hidden_layers</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">1</th>\n",
       "      <th>1</th>\n",
       "      <td>11.307692</td>\n",
       "      <td>0.999930</td>\n",
       "      <td>0.947564</td>\n",
       "      <td>0.900243</td>\n",
       "      <td>27.491425</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9.000000</td>\n",
       "      <td>0.999815</td>\n",
       "      <td>0.998428</td>\n",
       "      <td>0.997009</td>\n",
       "      <td>48.438291</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>11.379310</td>\n",
       "      <td>0.996620</td>\n",
       "      <td>0.911742</td>\n",
       "      <td>0.832329</td>\n",
       "      <td>129.095418</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  epoch  train_score  test_score  \\\n",
       "no_of_features hidden_layers                                       \n",
       "1              1              11.307692     0.999930    0.947564   \n",
       "               3               9.000000     0.999815    0.998428   \n",
       "               5              11.379310     0.996620    0.911742   \n",
       "\n",
       "                              test_score_20  time_taken  \n",
       "no_of_features hidden_layers                             \n",
       "1              1                   0.900243   27.491425  \n",
       "               3                   0.997009   48.438291  \n",
       "               5                   0.832329  129.095418  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pgb = past_scores.groupby(by=['no_of_features', 'hidden_layers'])\n",
    "pgb.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-23T21:43:47.568508Z",
     "start_time": "2017-06-23T21:43:47.553437Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>train_score</th>\n",
       "      <th>test_score</th>\n",
       "      <th>test_score_20</th>\n",
       "      <th>time_taken</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_of_features</th>\n",
       "      <th>hidden_layers</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">1</th>\n",
       "      <th>1</th>\n",
       "      <td>5.182812</td>\n",
       "      <td>0.000159</td>\n",
       "      <td>0.011661</td>\n",
       "      <td>0.022184</td>\n",
       "      <td>14.036460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.242641</td>\n",
       "      <td>0.000278</td>\n",
       "      <td>0.003207</td>\n",
       "      <td>0.006102</td>\n",
       "      <td>35.543510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>4.901995</td>\n",
       "      <td>0.006017</td>\n",
       "      <td>0.081695</td>\n",
       "      <td>0.155172</td>\n",
       "      <td>67.656089</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 epoch  train_score  test_score  \\\n",
       "no_of_features hidden_layers                                      \n",
       "1              1              5.182812     0.000159    0.011661   \n",
       "               3              4.242641     0.000278    0.003207   \n",
       "               5              4.901995     0.006017    0.081695   \n",
       "\n",
       "                              test_score_20  time_taken  \n",
       "no_of_features hidden_layers                             \n",
       "1              1                   0.022184   14.036460  \n",
       "               3                   0.006102   35.543510  \n",
       "               5                   0.155172   67.656089  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pgb.std()"
   ]
  }
 ],
 "metadata": {
  "_draft": {
   "nbviewer_url": "https://gist.github.com/7d1ace18a82178e15ece8fc5252fce88"
  },
  "anaconda-cloud": {},
  "gist": {
   "data": {
    "description": "Hyper parameter tuning",
    "public": false
   },
   "id": "7d1ace18a82178e15ece8fc5252fce88"
  },
  "kernelspec": {
   "display_name": "Python [conda env:p3]",
   "language": "python",
   "name": "conda-env-p3-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
